From 751a896d10759df229cb4dce9cc02f4be8e55e03 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Tue, 6 Oct 2026 16:59:52 +0200 Subject: [PATCH] LF line endings Basic, Vector, VectorF16 and VectorF32 were stored with CRLF; convert them to LF like the rest of the tree. No other change to their content. .gitattributes keeps text files LF from here on. Co-Authored-By: Claude Fable 5.1 --- .gitattributes | 1 + interfaces/Crafter.Math-Basic.cppm | 34 +- interfaces/Crafter.Math-Vector.cppm | 936 ++--- interfaces/Crafter.Math-VectorF16.cppm | 2648 +++++++------- interfaces/Crafter.Math-VectorF32.cppm | 4426 ++++++++++++------------ 5 files changed, 4023 insertions(+), 4022 deletions(-) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/interfaces/Crafter.Math-Basic.cppm b/interfaces/Crafter.Math-Basic.cppm index 2b38b45..cd16528 100755 --- a/interfaces/Crafter.Math-Basic.cppm +++ b/interfaces/Crafter.Math-Basic.cppm @@ -1,18 +1,18 @@ -//SPDX-License-Identifier: LGPL-3.0-only -//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® - -export module Crafter.Math:Basic; -import std; -import :VectorF32; - -namespace Crafter { - template - constexpr T ToRadian(T degrees) { - return degrees * (std::numbers::pi / 180); - } - - #if (defined(__x86_64) && !defined(__AVX512FP16__)) || defined(__riscv_vector) || !defined(__FLT16_MAX__) - export template - using VectorF16 = VectorF32; - #endif +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +export module Crafter.Math:Basic; +import std; +import :VectorF32; + +namespace Crafter { + template + constexpr T ToRadian(T degrees) { + return degrees * (std::numbers::pi / 180); + } + + #if (defined(__x86_64) && !defined(__AVX512FP16__)) || defined(__riscv_vector) || !defined(__FLT16_MAX__) + export template + using VectorF16 = VectorF32; + #endif } \ No newline at end of file diff --git a/interfaces/Crafter.Math-Vector.cppm b/interfaces/Crafter.Math-Vector.cppm index d8194df..6f23a88 100755 --- a/interfaces/Crafter.Math-Vector.cppm +++ b/interfaces/Crafter.Math-Vector.cppm @@ -1,469 +1,469 @@ -//SPDX-License-Identifier: LGPL-3.0-only -//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® - -export module Crafter.Math:Vector; -import std; - -namespace Crafter { - template - struct __attribute__((packed)) VectorBase { - T v[Aligment]; - }; - - template - struct __attribute__((packed)) VectorBase { - union { - T v[Aligment]; - T x, r; - }; - }; - - template - struct __attribute__((packed)) VectorBase { - union { - T v[Aligment]; - struct __attribute__((packed)) { - T x, y; - }; - struct __attribute__((packed)) { - T r, g; - }; - }; - constexpr VectorBase(T x, T y): x(x), y(y) { - - } - constexpr VectorBase() = default; - }; - - template - struct __attribute__((packed)) VectorBase { - union { - T v[Aligment]; - struct __attribute__((packed)) { - T x, y, z; - }; - struct __attribute__((packed)) { - T r, g, b; - }; - }; - constexpr VectorBase(T x, T y, T z): x(x), y(y), z(z) { - - } - constexpr VectorBase() = default; - }; - - template - struct __attribute__((packed)) VectorBase { - union { - T v[Aligment]; - struct __attribute__((packed)) { - T x, y, z, w; - }; - struct __attribute__((packed)) { - T r, g, b, a; - }; - }; - constexpr VectorBase(T x, T y, T z, T w): x(x), y(y), z(z), w(w) { - - } - constexpr VectorBase() = default; - }; - - template - struct VectorBase { - T v[Len]; - }; - - template - struct VectorBase { - union { - T v[1]; - T x, r; - }; - }; - - template - struct VectorBase { - union { - T v[2]; - struct { - T x, y; - }; - struct { - T r, g; - }; - }; - constexpr VectorBase(T x, T y): x(x), y(y) { - - } - constexpr VectorBase() = default; - }; - - template - struct VectorBase { - union { - T v[3]; - struct { - T x, y, z; - }; - struct { - T r, g, b; - }; - }; - constexpr VectorBase(T x, T y, T z): x(x), y(y), z(z) { - - } - constexpr VectorBase() = default; - }; - - template - struct VectorBase { - union { - T v[4]; - struct { - T x, y, z, w; - }; - struct { - T r, g, b, a; - }; - }; - constexpr VectorBase(T x, T y, T z, T w): x(x), y(y), z(z), w(w) { - - } - constexpr VectorBase() = default; - }; - - export template - class Vector : public VectorBase { - public: - constexpr Vector(T x, T y, T z, T w ) requires(std::same_as && Len == 4) : VectorBase(x, y, z, w) { - - } - constexpr Vector(T x, T y, T z) requires(std::same_as && Len == 3) : VectorBase(x, y, z) { - - } - constexpr Vector(T x, T y) requires(std::same_as && Len == 2) : VectorBase(x, y) { - - } - - constexpr Vector() = default; - - template - constexpr operator Vector() const { - Vector returnVector; - for(std::uint32_t i = 0; i < Len; i++) { - returnVector.v[i] = this->v[i]; - } - return returnVector; - } - - template - constexpr Vector operator+(Vector b) const { - Vector resultVector; - for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { - resultVector.v[i] = this->v[i]+b.v[i]; - } - return resultVector; - } - template - constexpr Vector operator-(Vector b) const { - Vector resultVector; - for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { - resultVector.v[i] = this->v[i]-b.v[i]; - } - return resultVector; - } - constexpr Vector operator-(){ - Vector resultVector; - for(std::uint32_t i = 0; i < Len; i++) { - resultVector.v[i] = -this->v[i]; - } - return resultVector; - } - template - constexpr Vector operator*(Vector b) const { - Vector resultVector; - for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { - resultVector.v[i] = this->v[i]*b.v[i]; - } - return resultVector; - } - template - constexpr Vector operator/(Vector b) const { - Vector resultVector; - for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { - resultVector.v[i] = this->v[i]/b.v[i]; - } - return resultVector; - } - template - constexpr void operator+=(Vector b){ - for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { - this->v[i]+=b.v[i]; - } - } - template - constexpr void operator-=(Vector b){ - for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { - this->v[i]-=b.v[i]; - } - } - template - constexpr void operator*=(Vector b){ - for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { - this->v[i]*=b.v[i]; - } - } - template - constexpr void operator/=(Vector b){ - for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { - this->v[i]/=b.v[i]; - } - } - - template - constexpr Vector operator+(BT b) const requires std::is_arithmetic_v { - Vector resultVector; - for(std::uint32_t i = 0; i < Len; i++) { - resultVector.v[i] = this->v[i] + b; - } - return resultVector; - } - template - constexpr Vector operator-(BT b) const requires std::is_arithmetic_v { - Vector resultVector; - for(std::uint32_t i = 0; i < Len; i++) { - resultVector.v[i] = this->v[i] - b; - } - return resultVector; - } - template - constexpr Vector operator*(BT b) const requires std::is_arithmetic_v { - Vector resultVector; - for(std::uint32_t i = 0; i < Len; i++) { - resultVector.v[i] = this->v[i] * b; - } - return resultVector; - } - template - constexpr Vector operator/(BT b) const requires std::is_arithmetic_v { - Vector resultVector; - for(std::uint32_t i = 0; i < Len; i++) { - resultVector.v[i] = this->v[i] / b; - } - return resultVector; - } - template - constexpr void operator+=(BT b) requires std::is_arithmetic_v{ - for(std::uint32_t i = 0; i < Len; i++) { - this->v[i] += b; - } - } - template - constexpr void operator-=(BT b) requires std::is_arithmetic_v{ - for(std::uint32_t i = 0; i < Len; i++) { - this->v[i] -= b; - } - } - template - constexpr void operator*=(BT b) requires std::is_arithmetic_v{ - for(std::uint32_t i = 0; i < Len; i++) { - this->v[i] *= b; - } - } - template - constexpr void operator/=(BT b) requires std::is_arithmetic_v{ - for(std::uint32_t i = 0; i < Len; i++) { - this->v[i] /= b; - } - } - - template - constexpr bool operator==(Vector b) const { - for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { - if(this->v[i] != b.v[i]) { - return false; - } - } - return true; - } - - template - constexpr bool operator!=(Vector b) const { - for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { - if(this->v[i] != b.v[i]) { - return true; - } - } - return false; - } - - constexpr void Normalize() { - T fLength = Length(); - - fLength = 1.0f / fLength; - - for(std::uint32_t i = 0; i < Len; i++) { - this->v[i] *= fLength; - } - } - - constexpr void Invert() { - for(std::uint32_t i = 0; i < Len; i++) { - this->v[i] = -this->v[i]; - } - } - - constexpr T Length() const { - T Result = LengthSq(); - return std::sqrtf(Result); - } - - constexpr T LengthSq() const { - return Dot(*this, *this); - } - - template - constexpr static Vector Cross(Vector a, Vector b) requires(Len == 3 && Alen >= 3 && Blen >= 3) { - return Vector( - (a.v[1] * b.v[2]) - (a.v[2] * b.v[1]), - (a.v[2] * b.v[0]) - (a.v[0] * b.v[2]), - (a.v[0] * b.v[1]) - (a.v[1] * b.v[0]) - ); - } - - template - constexpr static Vector Normalize(Vector a) requires(Len == Alen) { - Vector returned; - T fLength = a.Length(); - - if (fLength > 0) { - fLength = 1.0f / fLength; - } - - for(std::uint32_t i = 0; i < Len; i++) { - returned.v[i] = a.v[i] * fLength; - } - return returned; - } - - template - constexpr static T Dot(Vector a, Vector b) requires(Alen >= Len && Blen >= Len) { - T accumulate = a.v[0] * b.v[0]; - for(std::uint32_t i = 1; i < Len; i++) { - accumulate += a.v[i] * b.v[i]; - } - return accumulate; - } - - template - constexpr static Vector Rotate(Vector v, Vector q) requires(Len == 3) { - Vector qv(q.x, q.y, q.z); - Vector t = Vector::Cross(qv, v) * T(2); - return v + t * q.w + Vector::Cross(qv, t); - } - - template - constexpr static Vector RotatePivot(Vector v, Vector q, Vector pivot) requires(Len == 3) { - Vector translated = v - pivot; - Vector qv(q.x, q.y, q.z); - Vector t = Cross(qv, translated) * T(2); - Vector rotated = translated + t * q.w +Cross(qv, t); - return rotated + pivot; - } - - template - constexpr static Vector QuanternionFromBasis(Vector right, Vector up, Vector forward) requires(Len == 4) { - T m00 = right.x; - T m01 = up.x; - T m02 = forward.x; - - T m10 = right.y; - T m11 = up.y; - T m12 = forward.y; - - T m20 = right.z; - T m21 = up.z; - T m22 = forward.z; - - T trace = m00 + m11 + m22; - - Vector q; - - if (trace > std::numeric_limits::epsilon()) { - T s = std::sqrt(trace + T(1)) * T(2); - q.w = T(0.25) * s; - q.x = (m21 - m12) / s; - q.y = (m02 - m20) / s; - q.z = (m10 - m01) / s; - } - else if ((m00 > m11) && (m00 > m22)) { - T s = std::sqrt(T(1) + m00 - m11 - m22) * T(2); - q.w = (m21 - m12) / s; - q.x = T(0.25) * s; - q.y = (m01 + m10) / s; - q.z = (m02 + m20) / s; - } - else if (m11 > m22) { - T s = std::sqrt(T(1) + m11 - m00 - m22) * T(2); - q.w = (m02 - m20) / s; - q.x = (m01 + m10) / s; - q.y = T(0.25) * s; - q.z = (m12 + m21) / s; - } - else { - T s = std::sqrt(T(1) + m22 - m00 - m11) * T(2); - q.w = (m10 - m01) / s; - q.x = (m02 + m20) / s; - q.y = (m12 + m21) / s; - q.z = T(0.25) * s; - } - - q.Normalize(); - return q; - } - - constexpr static Vector QuanternionFromEuler(T rollHalf, T pitchHalf, T yawHalf) { - T cr = std::cos(rollHalf); - T sr = std::sin(rollHalf); - T cp = std::cos(pitchHalf); - T sp = std::sin(pitchHalf); - T cy = std::cos(yawHalf); - T sy = std::sin(yawHalf); - - return Vector( - sr * cp * cy - cr * sp * sy, - cr * sp * cy + sr * cp * sy, - cr * cp * sy - sr * sp * cy, - cr * cp * cy + sr * sp * sy - ); - } - }; -} - - - -template -struct std::formatter> : std::formatter { - auto format(const Crafter::Vector& vec, format_context& ctx) const { - std::string out = "{"; - for(std::uint32_t i2 = 0; i2 < Len; i2++) { - #ifdef __x86_64 - if constexpr(std::same_as) { - out += std::format("{}", static_cast(vec.v[i2])); - } else - #endif - { - out += std::format("{}", vec.v[i2]); - } - if (i2 + 1 < Len) out += ","; - } - out += "}"; - return std::formatter::format(out, ctx); - } -}; - -template -constexpr Crafter::Vector operator*(BT b, const Crafter::Vector& v) requires std::is_arithmetic_v { - return v * b; +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +export module Crafter.Math:Vector; +import std; + +namespace Crafter { + template + struct __attribute__((packed)) VectorBase { + T v[Aligment]; + }; + + template + struct __attribute__((packed)) VectorBase { + union { + T v[Aligment]; + T x, r; + }; + }; + + template + struct __attribute__((packed)) VectorBase { + union { + T v[Aligment]; + struct __attribute__((packed)) { + T x, y; + }; + struct __attribute__((packed)) { + T r, g; + }; + }; + constexpr VectorBase(T x, T y): x(x), y(y) { + + } + constexpr VectorBase() = default; + }; + + template + struct __attribute__((packed)) VectorBase { + union { + T v[Aligment]; + struct __attribute__((packed)) { + T x, y, z; + }; + struct __attribute__((packed)) { + T r, g, b; + }; + }; + constexpr VectorBase(T x, T y, T z): x(x), y(y), z(z) { + + } + constexpr VectorBase() = default; + }; + + template + struct __attribute__((packed)) VectorBase { + union { + T v[Aligment]; + struct __attribute__((packed)) { + T x, y, z, w; + }; + struct __attribute__((packed)) { + T r, g, b, a; + }; + }; + constexpr VectorBase(T x, T y, T z, T w): x(x), y(y), z(z), w(w) { + + } + constexpr VectorBase() = default; + }; + + template + struct VectorBase { + T v[Len]; + }; + + template + struct VectorBase { + union { + T v[1]; + T x, r; + }; + }; + + template + struct VectorBase { + union { + T v[2]; + struct { + T x, y; + }; + struct { + T r, g; + }; + }; + constexpr VectorBase(T x, T y): x(x), y(y) { + + } + constexpr VectorBase() = default; + }; + + template + struct VectorBase { + union { + T v[3]; + struct { + T x, y, z; + }; + struct { + T r, g, b; + }; + }; + constexpr VectorBase(T x, T y, T z): x(x), y(y), z(z) { + + } + constexpr VectorBase() = default; + }; + + template + struct VectorBase { + union { + T v[4]; + struct { + T x, y, z, w; + }; + struct { + T r, g, b, a; + }; + }; + constexpr VectorBase(T x, T y, T z, T w): x(x), y(y), z(z), w(w) { + + } + constexpr VectorBase() = default; + }; + + export template + class Vector : public VectorBase { + public: + constexpr Vector(T x, T y, T z, T w ) requires(std::same_as && Len == 4) : VectorBase(x, y, z, w) { + + } + constexpr Vector(T x, T y, T z) requires(std::same_as && Len == 3) : VectorBase(x, y, z) { + + } + constexpr Vector(T x, T y) requires(std::same_as && Len == 2) : VectorBase(x, y) { + + } + + constexpr Vector() = default; + + template + constexpr operator Vector() const { + Vector returnVector; + for(std::uint32_t i = 0; i < Len; i++) { + returnVector.v[i] = this->v[i]; + } + return returnVector; + } + + template + constexpr Vector operator+(Vector b) const { + Vector resultVector; + for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { + resultVector.v[i] = this->v[i]+b.v[i]; + } + return resultVector; + } + template + constexpr Vector operator-(Vector b) const { + Vector resultVector; + for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { + resultVector.v[i] = this->v[i]-b.v[i]; + } + return resultVector; + } + constexpr Vector operator-(){ + Vector resultVector; + for(std::uint32_t i = 0; i < Len; i++) { + resultVector.v[i] = -this->v[i]; + } + return resultVector; + } + template + constexpr Vector operator*(Vector b) const { + Vector resultVector; + for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { + resultVector.v[i] = this->v[i]*b.v[i]; + } + return resultVector; + } + template + constexpr Vector operator/(Vector b) const { + Vector resultVector; + for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { + resultVector.v[i] = this->v[i]/b.v[i]; + } + return resultVector; + } + template + constexpr void operator+=(Vector b){ + for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { + this->v[i]+=b.v[i]; + } + } + template + constexpr void operator-=(Vector b){ + for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { + this->v[i]-=b.v[i]; + } + } + template + constexpr void operator*=(Vector b){ + for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { + this->v[i]*=b.v[i]; + } + } + template + constexpr void operator/=(Vector b){ + for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { + this->v[i]/=b.v[i]; + } + } + + template + constexpr Vector operator+(BT b) const requires std::is_arithmetic_v { + Vector resultVector; + for(std::uint32_t i = 0; i < Len; i++) { + resultVector.v[i] = this->v[i] + b; + } + return resultVector; + } + template + constexpr Vector operator-(BT b) const requires std::is_arithmetic_v { + Vector resultVector; + for(std::uint32_t i = 0; i < Len; i++) { + resultVector.v[i] = this->v[i] - b; + } + return resultVector; + } + template + constexpr Vector operator*(BT b) const requires std::is_arithmetic_v { + Vector resultVector; + for(std::uint32_t i = 0; i < Len; i++) { + resultVector.v[i] = this->v[i] * b; + } + return resultVector; + } + template + constexpr Vector operator/(BT b) const requires std::is_arithmetic_v { + Vector resultVector; + for(std::uint32_t i = 0; i < Len; i++) { + resultVector.v[i] = this->v[i] / b; + } + return resultVector; + } + template + constexpr void operator+=(BT b) requires std::is_arithmetic_v{ + for(std::uint32_t i = 0; i < Len; i++) { + this->v[i] += b; + } + } + template + constexpr void operator-=(BT b) requires std::is_arithmetic_v{ + for(std::uint32_t i = 0; i < Len; i++) { + this->v[i] -= b; + } + } + template + constexpr void operator*=(BT b) requires std::is_arithmetic_v{ + for(std::uint32_t i = 0; i < Len; i++) { + this->v[i] *= b; + } + } + template + constexpr void operator/=(BT b) requires std::is_arithmetic_v{ + for(std::uint32_t i = 0; i < Len; i++) { + this->v[i] /= b; + } + } + + template + constexpr bool operator==(Vector b) const { + for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { + if(this->v[i] != b.v[i]) { + return false; + } + } + return true; + } + + template + constexpr bool operator!=(Vector b) const { + for(std::uint32_t i = 0; i < std::min(Len, Blen); i++) { + if(this->v[i] != b.v[i]) { + return true; + } + } + return false; + } + + constexpr void Normalize() { + T fLength = Length(); + + fLength = 1.0f / fLength; + + for(std::uint32_t i = 0; i < Len; i++) { + this->v[i] *= fLength; + } + } + + constexpr void Invert() { + for(std::uint32_t i = 0; i < Len; i++) { + this->v[i] = -this->v[i]; + } + } + + constexpr T Length() const { + T Result = LengthSq(); + return std::sqrtf(Result); + } + + constexpr T LengthSq() const { + return Dot(*this, *this); + } + + template + constexpr static Vector Cross(Vector a, Vector b) requires(Len == 3 && Alen >= 3 && Blen >= 3) { + return Vector( + (a.v[1] * b.v[2]) - (a.v[2] * b.v[1]), + (a.v[2] * b.v[0]) - (a.v[0] * b.v[2]), + (a.v[0] * b.v[1]) - (a.v[1] * b.v[0]) + ); + } + + template + constexpr static Vector Normalize(Vector a) requires(Len == Alen) { + Vector returned; + T fLength = a.Length(); + + if (fLength > 0) { + fLength = 1.0f / fLength; + } + + for(std::uint32_t i = 0; i < Len; i++) { + returned.v[i] = a.v[i] * fLength; + } + return returned; + } + + template + constexpr static T Dot(Vector a, Vector b) requires(Alen >= Len && Blen >= Len) { + T accumulate = a.v[0] * b.v[0]; + for(std::uint32_t i = 1; i < Len; i++) { + accumulate += a.v[i] * b.v[i]; + } + return accumulate; + } + + template + constexpr static Vector Rotate(Vector v, Vector q) requires(Len == 3) { + Vector qv(q.x, q.y, q.z); + Vector t = Vector::Cross(qv, v) * T(2); + return v + t * q.w + Vector::Cross(qv, t); + } + + template + constexpr static Vector RotatePivot(Vector v, Vector q, Vector pivot) requires(Len == 3) { + Vector translated = v - pivot; + Vector qv(q.x, q.y, q.z); + Vector t = Cross(qv, translated) * T(2); + Vector rotated = translated + t * q.w +Cross(qv, t); + return rotated + pivot; + } + + template + constexpr static Vector QuanternionFromBasis(Vector right, Vector up, Vector forward) requires(Len == 4) { + T m00 = right.x; + T m01 = up.x; + T m02 = forward.x; + + T m10 = right.y; + T m11 = up.y; + T m12 = forward.y; + + T m20 = right.z; + T m21 = up.z; + T m22 = forward.z; + + T trace = m00 + m11 + m22; + + Vector q; + + if (trace > std::numeric_limits::epsilon()) { + T s = std::sqrt(trace + T(1)) * T(2); + q.w = T(0.25) * s; + q.x = (m21 - m12) / s; + q.y = (m02 - m20) / s; + q.z = (m10 - m01) / s; + } + else if ((m00 > m11) && (m00 > m22)) { + T s = std::sqrt(T(1) + m00 - m11 - m22) * T(2); + q.w = (m21 - m12) / s; + q.x = T(0.25) * s; + q.y = (m01 + m10) / s; + q.z = (m02 + m20) / s; + } + else if (m11 > m22) { + T s = std::sqrt(T(1) + m11 - m00 - m22) * T(2); + q.w = (m02 - m20) / s; + q.x = (m01 + m10) / s; + q.y = T(0.25) * s; + q.z = (m12 + m21) / s; + } + else { + T s = std::sqrt(T(1) + m22 - m00 - m11) * T(2); + q.w = (m10 - m01) / s; + q.x = (m02 + m20) / s; + q.y = (m12 + m21) / s; + q.z = T(0.25) * s; + } + + q.Normalize(); + return q; + } + + constexpr static Vector QuanternionFromEuler(T rollHalf, T pitchHalf, T yawHalf) { + T cr = std::cos(rollHalf); + T sr = std::sin(rollHalf); + T cp = std::cos(pitchHalf); + T sp = std::sin(pitchHalf); + T cy = std::cos(yawHalf); + T sy = std::sin(yawHalf); + + return Vector( + sr * cp * cy - cr * sp * sy, + cr * sp * cy + sr * cp * sy, + cr * cp * sy - sr * sp * cy, + cr * cp * cy + sr * sp * sy + ); + } + }; +} + + + +template +struct std::formatter> : std::formatter { + auto format(const Crafter::Vector& vec, format_context& ctx) const { + std::string out = "{"; + for(std::uint32_t i2 = 0; i2 < Len; i2++) { + #ifdef __x86_64 + if constexpr(std::same_as) { + out += std::format("{}", static_cast(vec.v[i2])); + } else + #endif + { + out += std::format("{}", vec.v[i2]); + } + if (i2 + 1 < Len) out += ","; + } + out += "}"; + return std::formatter::format(out, ctx); + } +}; + +template +constexpr Crafter::Vector operator*(BT b, const Crafter::Vector& v) requires std::is_arithmetic_v { + return v * b; } \ No newline at end of file diff --git a/interfaces/Crafter.Math-VectorF16.cppm b/interfaces/Crafter.Math-VectorF16.cppm index fc050b4..13f0bc9 100755 --- a/interfaces/Crafter.Math-VectorF16.cppm +++ b/interfaces/Crafter.Math-VectorF16.cppm @@ -1,1325 +1,1325 @@ -//SPDX-License-Identifier: LGPL-3.0-only -//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® - -module; -#ifdef __x86_64 -#include -#endif -export module Crafter.Math:VectorF16; -import std; -import :Common; - -// A real VectorF16 struct is provided only when _Float16 is available AND we -// are either on an x86_64 build with AVX512FP16 (intrinsic impl below) or on a -// non-x86_64 target (scalar fallback further below). On x86_64 without -// AVX512FP16 or anywhere without _Float16, Crafter.Math:Basic aliases -// VectorF16 to VectorF32 instead. -#ifdef __AVX512FP16__ -namespace Crafter { - export template - struct VectorF16 : public VectorBase { - template - friend struct VectorF16; - - constexpr VectorF16() = default; - constexpr VectorF16(VectorBase::VectorType v) { - this->v = v; - } - constexpr VectorF16(const _Float16* vB) { - Load(vB); - }; - constexpr VectorF16(_Float16 val) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - this->v = _mm_set1_ph(val); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - this->v = _mm256_set1_ph(val); - } else { - this->v = _mm512_set1_ph(val); - } - }; - constexpr void Load(const _Float16* vB) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - this->v = _mm_loadu_ph(vB); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - this->v = _mm256_loadu_ph(vB); - } else { - this->v = _mm512_loadu_ph(vB); - } - } - constexpr void Store(_Float16* vB) const { - if constexpr(std::is_same_v::VectorType, __m128h>) { - _mm_storeu_ph(vB, this->v); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - _mm256_storeu_ph(vB, this->v); - } else { - _mm512_storeu_ph(vB, this->v); - } - } - - template - constexpr std::array<_Float16, VectorBase::AlignmentElement> Store() const { - std::array<_Float16, VectorBase::AlignmentElement> returnArray; - Store(returnArray.data()); - return returnArray; - } - - template - constexpr operator VectorF16() const { - if constexpr (Len == BLen) { - if constexpr(std::is_same_v::VectorType, __m256h> && std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm256_castph256_ph128(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m512h> && std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm512_castph512_ph128(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m512h> && std::is_same_v::VectorType, __m256h>) { - return VectorF16(_mm512_castph512_ph256(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m128h> && std::is_same_v::VectorType, __m256h>) { - return VectorF16(_mm256_castph128_ph256(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m128h> && std::is_same_v::VectorType, __m512h>) { - return VectorF16(_mm512_castph128_ph512(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m256h> && std::is_same_v::VectorType, __m512h>) { - return VectorF16(_mm512_castph256_ph512(this->v)); - } else { - return VectorF16(this->v); - } - } else if constexpr (BLen <= Len) { - return this->template ExtractLo(); - } else { - if constexpr(std::is_same_v::VectorType, __m128h>) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - constexpr std::array::Alignment> shuffleMask = VectorBase::template GetExtractLoMaskEpi8(); - __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); - return VectorF16(_mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(this->v), shuffleVec))); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - constexpr std::array::AlignmentElement> permMask =VectorBase::template GetExtractLoMaskEpi16(); - __m256i permIdx = _mm256_loadu_epi16(permMask.data()); - __m256i result = _mm256_permutexvar_epi16(permIdx, _mm_castph_si256(this->v)); - return VectorF16(_mm_castsi128_ph(_mm256_castsi256_si128(result))); - } else { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); - __m512i permIdx = _mm512_loadu_epi16(permMask.data()); - __m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castph_si512(this->v)); - return VectorF16(_mm_castsi128_ph(_mm512_castsi512_si128(result))); - } - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); - __m256i permIdx = _mm256_loadu_epi16(permMask.data()); - __m256i result = _mm256_permutexvar_epi16(permIdx, _mm256_castsi128_si256(_mm_castph_si128(this->v))); - return VectorF16(_mm256_castsi256_ph(result)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); - __m256i permIdx = _mm256_loadu_epi16(permMask.data()); - __m256i result = _mm256_permutexvar_epi16(permIdx, _mm256_castph_si256(this->v)); - return VectorF16(_mm256_castsi256_ph(result)); - } else { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); - __m256i permIdx = _mm512_loadu_epi16(permMask.data()); - __m256i result = _mm512_permutexvar_epi16(permIdx, _mm512_castsi512_si256(_mm512_castph_si512(this->v))); - return VectorF16(_mm256_castsi256_ph(result)); - } - } else { - if constexpr(std::is_same_v::VectorType, __m128h>) { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); - __m512i permIdx = _mm512_loadu_epi16(permMask.data()); - __m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castsi128_si512(_mm_castph_si128(this->v))); - return VectorF16(_mm512_castsi512_ph(result)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); - __m512i permIdx = _mm512_loadu_epi16(permMask.data()); - __m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castsi256_si512(_mm256_castph_si256(this->v))); - return VectorF16(_mm512_castsi512_ph(result)); - } else { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); - __m512i permIdx = _mm512_loadu_epi16(permMask.data()); - __m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castph_si512(this->v)); - return VectorF16(_mm512_castsi512_ph(result)); - } - } - } - } - - constexpr VectorF16 operator+(VectorF16 b) const { - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm_add_ph(this->v, b.v)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16(_mm256_add_ph(this->v, b.v)); - } else { - return VectorF16(_mm512_add_ph(this->v, b.v)); - } - } - - constexpr VectorF16 operator-(VectorF16 b) const { - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm_sub_ph(this->v, b.v)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16(_mm256_sub_ph(this->v, b.v)); - } else { - return VectorF16(_mm512_sub_ph(this->v, b.v)); - } - } - - constexpr VectorF16 operator*(VectorF16 b) const { - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm_mul_ph(this->v, b.v)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16(_mm256_mul_ph(this->v, b.v)); - } else { - return VectorF16(_mm512_mul_ph(this->v, b.v)); - } - } - - constexpr VectorF16 operator/(VectorF16 b) const { - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm_div_ph(this->v, b.v)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16(_mm256_div_ph(this->v, b.v)); - } else { - return VectorF16(_mm512_div_ph(this->v, b.v)); - } - } - - - constexpr void operator+=(VectorF16 b) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - this->v = _mm_add_ph(this->v, b.v); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - this->v = _mm256_add_ph(this->v, b.v); - } else { - this->v = _mm512_add_ph(this->v, b.v); - } - } - - constexpr void operator-=(VectorF16 b) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - this->v = _mm_sub_ph(this->v, b.v); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - this->v = _mm256_sub_ph(this->v, b.v); - } else { - this->v = _mm512_sub_ph(this->v, b.v); - } - } - - constexpr void operator*=(VectorF16 b) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - this->v = _mm_mul_ph(this->v, b.v); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - this->v = _mm256_mul_ph(this->v, b.v); - } else { - this->v = _mm512_mul_ph(this->v, b.v); - } - } - - constexpr void operator/=(VectorF16 b) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - this->v = _mm_div_ph(this->v, b.v); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - this->v = _mm256_div_ph(this->v, b.v); - } else { - this->v = _mm512_div_ph(this->v, b.v); - } - } - - constexpr VectorF16 operator+(_Float16 b) { - VectorF16 vB(b); - return *this + vB; - } - - constexpr VectorF16 operator-(_Float16 b) { - VectorF16 vB(b); - return *this - vB; - } - - constexpr VectorF16 operator*(_Float16 b) { - VectorF16 vB(b); - return *this * vB; - } - - constexpr VectorF16 operator/(_Float16 b) { - VectorF16 vB(b); - return *this / vB; - } - - constexpr void operator+=(_Float16 b) { - VectorF16 vB(b); - *this += vB; - } - - constexpr void operator-=(_Float16 b) { - VectorF16 vB(b); - *this -= vB; - } - - constexpr void operator*=(_Float16 b) { - VectorF16 vB(b); - *this *= vB; - } - - constexpr void operator/=(_Float16 b) { - VectorF16 vB(b); - *this /= vB; - } - - constexpr VectorF16 operator-(){ - return Negate::GetAllTrue()>(); - } - - constexpr bool operator==(VectorF16 b) const { - if constexpr(std::is_same_v::VectorType, __m128h>) { - return _mm_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) == 255; - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return _mm256_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) == 65535; - } else { - return _mm512_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) == 4294967295; - } - } - - constexpr bool operator!=(VectorF16 b) const { - if constexpr(std::is_same_v::VectorType, __m128h>) { - return _mm_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) != 255; - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return _mm256_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) != 65535; - } else { - return _mm512_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) != 4294967295; - } - } - - template - constexpr VectorF16 ExtractLo() const { - if constexpr(Packing > 1) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - constexpr std::array::Alignment> shuffleMask = VectorBase::template GetExtractLoMaskEpi8(); - __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); - return VectorF16(_mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(this->v), shuffleVec))); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); - __m256i permIdx = _mm256_loadu_epi16(permMask.data()); - __m256i result = _mm256_permutexvar_epi16(permIdx, _mm256_castph_si256(this->v)); - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm256_castph256_ph128(_mm256_castsi256_ph(result))); - } else { - return VectorF16(_mm256_castsi256_ph(result)); - } - } else { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); - __m512i permIdx = _mm512_loadu_epi16(permMask.data()); - __m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castph_si512(this->v)); - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm512_castph512_ph128(_mm512_castsi512_ph(result))); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16(_mm512_castph512_ph256(_mm512_castsi512_ph(result))); - } else { - return VectorF16(_mm512_castsi512_ph(result)); - } - } - } else { - if constexpr(std::is_same_v::VectorType, __m256h> && std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm256_castph256_ph128(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m512h> && std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm512_castph512_ph128(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m512h> && std::is_same_v::VectorType, __m256h>) { - return VectorF16(_mm512_castph512_ph256(this->v)); - } else { - return VectorF16(this->v); - } - } - } - - constexpr VectorF16 Cos() { - if constexpr (std::is_same_v::VectorType, __m128h>) { - __m256 wide = _mm256_cvtph_ps(_mm_castph_si128(this->v)); - wide = VectorBase::cos_f32x8(wide); - return VectorF16( - _mm_castsi128_ph(_mm256_cvtps_ph(wide, _MM_FROUND_TO_NEAREST_INT))); - - } else if constexpr (std::is_same_v::VectorType, __m256h>) { - __m512 wide = _mm512_cvtph_ps(_mm256_castph_si256(this->v)); - wide = VectorBase::cos_f32x16(wide); - return VectorF16( - _mm256_castsi256_ph(_mm512_cvtps_ph(wide, _MM_FROUND_TO_NEAREST_INT))); - - } else { - __m256i lo = _mm512_castsi512_si256(_mm512_castph_si512(this->v)); - __m256i hi = _mm512_extracti64x4_epi64(_mm512_castph_si512(this->v), 1); - __m512i cosLo =VectorBase::cos_f32x16(_mm512_cvtph_ps(lo)); - __m512i cosHi =VectorBase::cos_f32x16(_mm512_cvtph_ps(hi)); - __m256i lo_ph = _mm512_cvtps_ph(cosLo, _MM_FROUND_TO_NEAREST_INT); - __m256i hi_ph = _mm512_cvtps_ph(cosHi, _MM_FROUND_TO_NEAREST_INT); - return VectorF16(_mm512_castsi512_ph(_mm512_inserti64x4(_mm512_castsi256_si512(lo_ph), hi_ph, 1))); - } - } - - constexpr VectorF16 Sin() { - if constexpr (std::is_same_v::VectorType, __m128h>) { - __m256 wide = _mm256_cvtph_ps(_mm_castph_si128(this->v)); - wide = VectorBase::sin_f32x8(wide); - return VectorF16(_mm_castsi128_ph(_mm256_cvtps_ph(wide, _MM_FROUND_TO_NEAREST_INT))); - - } else if constexpr (std::is_same_v::VectorType, __m256h>) { - __m512 wide = _mm512_cvtph_ps(_mm256_castph_si256(this->v)); - wide = VectorBase::sin_f32x16(wide); - return VectorF16(_mm256_castsi256_ph(_mm512_cvtps_ph(wide, _MM_FROUND_TO_NEAREST_INT))); - - } else { - __m256i lo = _mm512_castsi512_si256(_mm512_castph_si512(this->v)); - __m256i hi = _mm512_extracti64x4_epi64(_mm512_castph_si512(this->v), 1); - __m512i loSin = VectorBase::sin_f32x16(_mm512_cvtph_ps(lo)); - __m512i hiSin = VectorBase::sin_f32x16(_mm512_cvtph_ps(hi)); - __m256i lo_ph = _mm512_cvtps_ph(loSin, _MM_FROUND_TO_NEAREST_INT); - __m256i hi_ph = _mm512_cvtps_ph(hiSin, _MM_FROUND_TO_NEAREST_INT); - return VectorF16(_mm512_castsi512_ph(_mm512_inserti64x4(_mm512_castsi256_si512(lo_ph), hi_ph, 1))); - } - } - - std::tuple, VectorF16> SinCos() { - if constexpr (std::is_same_v::VectorType, __m128h>) { - __m256 wide = _mm256_cvtph_ps(_mm_castph_si128(this->v)); - __m256 s, c; - VectorBase::sincos_f32x8(wide, s, c); - return { - VectorF16(_mm_castsi128_ph(_mm256_cvtps_ph(s, _MM_FROUND_TO_NEAREST_INT))), - VectorF16(_mm_castsi128_ph(_mm256_cvtps_ph(c, _MM_FROUND_TO_NEAREST_INT))) - }; - - } else if constexpr (std::is_same_v::VectorType, __m256h>) { - __m512 wide = _mm512_cvtph_ps(_mm256_castph_si256(this->v)); - __m512 s, c; - VectorBase::sincos_f32x16(wide, s, c); - return { - VectorF16(_mm256_castsi256_ph(_mm512_cvtps_ph(s, _MM_FROUND_TO_NEAREST_INT))), - VectorF16(_mm256_castsi256_ph(_mm512_cvtps_ph(c, _MM_FROUND_TO_NEAREST_INT))) - }; - - } else { - __m256i lo = _mm512_castsi512_si256(_mm512_castph_si512(this->v)); - __m256i hi = _mm512_extracti64x4_epi64(_mm512_castph_si512(this->v), 1); - - __m512 s_lo, c_lo, s_hi, c_hi; - VectorBase::sincos_f32x16(_mm512_cvtph_ps(lo), s_lo, c_lo); - VectorBase::sincos_f32x16(_mm512_cvtph_ps(hi), s_hi, c_hi); - - auto pack = [](__m256i lo_ph, __m256i hi_ph) { - return _mm512_castsi512_ph(_mm512_inserti64x4(_mm512_castsi256_si512(lo_ph), hi_ph, 1)); - }; - return { - VectorF16(pack(_mm512_cvtps_ph(s_lo, _MM_FROUND_TO_NEAREST_INT), _mm512_cvtps_ph(s_hi, _MM_FROUND_TO_NEAREST_INT))), - VectorF16(pack( _mm512_cvtps_ph(c_lo, _MM_FROUND_TO_NEAREST_INT), _mm512_cvtps_ph(c_hi, _MM_FROUND_TO_NEAREST_INT))) - }; - } - } - - template values> - constexpr VectorF16 Negate() { - std::array<_Float16, VectorBase::AlignmentElement> mask = VectorBase::template GetNegateMask(); - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm_castsi128_ph(_mm_xor_si128(_mm_castph_si128(this->v), _mm_loadu_epi16(mask.data())))); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16(_mm256_castsi256_ph(_mm256_xor_si256(_mm256_castph_si256(this->v), _mm256_loadu_epi16(mask.data())))); - } else { - return VectorF16(_mm512_castsi512_ph(_mm512_xor_si512(_mm512_castph_si512(this->v), _mm512_loadu_epi16(mask.data())))); - } - } - - static constexpr VectorF16 MulitplyAdd(VectorF16 a, VectorF16 b, VectorF16 add) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm_fmadd_ph(a.v, b.v, add.v)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16(_mm256_fmadd_ph(a.v, b.v, add.v)); - } else { - return VectorF16(_mm512_fmadd_ph(a.v, b.v, add.v)); - } - } - - static constexpr VectorF16 MulitplySub(VectorF16 a, VectorF16 b, VectorF16 sub) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm_fmsub_ph(a.v, b.v, sub.v)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16(_mm256_fmsub_ph(a.v, b.v, sub.v)); - } else { - return VectorF16(_mm512_fmsub_ph(a.v, b.v, sub.v)); - } - } - - constexpr static VectorF16 Cross(VectorF16 a, VectorF16 b) requires(Len == 3) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - constexpr std::array::AlignmentElement*2> shuffleMask1 = VectorBase::template GetShuffleMaskEpi8<{{1,2,0}}>(); - __m128i shuffleVec1 = _mm_loadu_epi8(shuffleMask1.data()); - __m128h row1 = _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(a.v), shuffleVec1)); - __m128h row4 = _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(b.v), shuffleVec1)); - - constexpr std::array::AlignmentElement*2> shuffleMask3 = VectorBase::template GetShuffleMaskEpi8<{{2,0,1}}>(); - __m128i shuffleVec3 = _mm_loadu_epi8(shuffleMask3.data()); - __m128h row3 = _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(a.v), shuffleVec3)); - __m128h row2 = _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(b.v), shuffleVec3)); - - __m128h result = _mm_mul_ph(row3, row4); - return _mm_fmsub_ph(row1,row2,result); - } else if constexpr (std::is_same_v::VectorType, __m256h>) { - constexpr std::array::AlignmentElement*2> shuffleMask1 = VectorBase::template GetShuffleMaskEpi8<{{1,2,0}}>(); - __m512i shuffleVec1 = _mm512_castsi256_si512(_mm256_loadu_epi8(shuffleMask1.data())); - __m256h row1 = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(a.v)), shuffleVec1))); - __m256h row4 = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(b.v)), shuffleVec1))); - - constexpr std::array::AlignmentElement*2> shuffleMask3 = VectorBase::template GetShuffleMaskEpi8<{{2,0,1}}>(); - - __m512i shuffleVec3 = _mm512_castsi256_si512(_mm256_loadu_epi8(shuffleMask3.data())); - __m256h row3 = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(a.v)), shuffleVec3))); - __m256h row2 = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(b.v)), shuffleVec3))); - - __m256h result = _mm256_mul_ph(row3, row4); - return _mm256_fmsub_ph(row1,row2,result); - } else { - constexpr std::array::AlignmentElement*2> shuffleMask1 = VectorBase::template GetShuffleMaskEpi8<{{1,2,0}}>(); - - __m512i shuffleVec1 = _mm512_loadu_epi8(shuffleMask1.data()); - __m512h row1 = _mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(a.v), shuffleVec1)); - __m512h row4 = _mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(b.v), shuffleVec1)); - - constexpr std::array::AlignmentElement*2> shuffleMask3 = VectorBase::template GetShuffleMaskEpi8<{{2,0,1}}>(); - - __m512i shuffleVec3 = _mm512_loadu_epi8(shuffleMask3.data()); - __m512h row3 = _mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(a.v), shuffleVec3)); - __m512h row2 = _mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(b.v), shuffleVec3)); - - __m512h result = _mm512_mul_ph(row3, row4); - return _mm512_fmsub_ph(row1,row2,result); - } - } - - constexpr static _Float16 Dot(VectorF16 a, VectorF16 b) requires(Packing == 1) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - __m128h mul = _mm_mul_ph(a.v, b.v); - return _mm_reduce_add_ph(mul); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - static_assert(std::is_same_v::VectorType>, "a.v is NOT VectorBase::VectorType"); - __m256h mul = _mm256_mul_ph(a.v, b.v); - return _mm256_reduce_add_ph(mul); - } else { - __m512h mul = _mm512_mul_ph(a.v, b.v); - return _mm512_reduce_add_ph(mul); - } - } - - template ShuffleValues> - constexpr VectorF16 Shuffle() { - if constexpr(VectorBase::template CheckEpi32Shuffle()) { - constexpr std::uint8_t imm = VectorBase::template GetShuffleMaskEpi32(); - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16(_mm_castsi128_ph(_mm_shuffle_epi32(_mm_castph_si128(this->v), imm))); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16(_mm256_castsi256_ph(_mm256_shuffle_epi32(_mm256_castph_si256(this->v), imm))); - } else { - return VectorF16(_mm512_castsi512_ph(_mm512_shuffle_epi32(_mm512_castph_si512(this->v), imm))); - } - } else if constexpr(VectorBase::template CheckEpi8Shuffle()){ - constexpr std::array::Alignment> shuffleMask = VectorBase::template GetShuffleMaskEpi8(); - if constexpr(std::is_same_v::VectorType, __m128h>) { - __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); - return VectorF16(_mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(this->v), shuffleVec))); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - __m256i shuffleVec = _mm256_loadu_epi8(shuffleMask.data()); - return VectorF16(_mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(this->v)), _mm512_castsi256_si512(shuffleVec))))); - } else { - __m512i shuffleVec = _mm512_loadu_epi8(shuffleMask.data()); - return VectorF16(_mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(this->v), shuffleVec))); - } - } else { - if constexpr(std::is_same_v::VectorType, __m128h>) { - constexpr std::array::Alignment> shuffleMask = VectorBase::template GetShuffleMaskEpi8(); - __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); - return VectorF16(_mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(this->v), shuffleVec))); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetPermuteMaskEpi16(); - __m256i permIdx = _mm256_loadu_epi16(permMask.data()); - return VectorF16(_mm256_castsi256_ph(_mm256_permutexvar_epi16(permIdx, _mm256_castph_si256(this->v)))); - } else { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetPermuteMaskEpi16(); - __m512i permIdx = _mm512_loadu_epi16(permMask.data()); - return VectorF16(_mm512_castsi512_ph(_mm512_permutexvar_epi16(permIdx, _mm512_castph_si512(this->v)))); - } - } - } - - // Public variadic surface — one name per op, arity locked to BatchSize - // (or 2*BatchSize for Dot). Forwards to the *Pack helpers below which - // carry the SIMD bodies and per-(Len,Packing) requires clauses. - template - requires ((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto Normalize(VectorF16 first, Rest... rest) { - return NormalizePack(first, rest...); - } - - template - requires ((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto Length(VectorF16 first, Rest... rest) { - return LengthPack(first, rest...); - } - - template - requires ((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto LengthSq(VectorF16 first, Rest... rest) { - return LengthSqPack(first, rest...); - } - - template - requires ((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == 2 * VectorBase::BatchSize)) - constexpr static auto Dot(VectorF16 first, Rest... rest) { - return DotPack(first, rest...); - } - - private: - constexpr static std::array, VectorBase::BatchSize> NormalizePack( - VectorF16 A, - VectorF16 C, - VectorF16 E, - VectorF16 G - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - VectorF16<1, 8> lenght = LengthNoShuffle(A, E, C, G); - constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1}; - __m128h one = _mm_loadu_ph(oneArr); - VectorF16<8, 1> fLenght(_mm_div_ph(one, lenght.v)); - - VectorF16<8, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,1,1,1,1}}>(); - VectorF16<8, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,3,3,3,3}}>(); - VectorF16<8, 1> fLenghtE = fLenght.template Shuffle<{{4,4,4,4,5,5,5,5}}>(); - VectorF16<8, 1> fLenghtG = fLenght.template Shuffle<{{6,6,6,6,7,7,7,7}}>(); - - return { - _mm_mul_ph(A.v, fLenghtA.v), - _mm_mul_ph(C.v, fLenghtC.v), - _mm_mul_ph(E.v, fLenghtE.v), - _mm_mul_ph(G.v, fLenghtG.v) - }; - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - VectorF16<1, 16> lenght = LengthNoShuffle(A, E, C, G); - constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; - __m256h one = _mm256_loadu_ph(oneArr); - VectorF16<16, 1> fLenght(_mm256_div_ph(one, lenght.v)); - - - VectorF16<16, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,1,1,1,1,8,8,8,8,9,9,9,9}}>(); - VectorF16<16, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,3,3,3,3,10,10,10,10,11,11,11,11}}>(); - VectorF16<16, 1> fLenghtE = fLenght.template Shuffle<{{4,4,4,4,5,5,5,5,12,12,12,12,13,13,13,13}}>(); - VectorF16<16, 1> fLenghtG = fLenght.template Shuffle<{{6,6,6,6,7,7,7,7,14,14,14,14,15,15,15,15}}>(); - - return { - _mm256_mul_ph(A.v, fLenghtA.v), - _mm256_mul_ph(C.v, fLenghtC.v), - _mm256_mul_ph(E.v, fLenghtE.v), - _mm256_mul_ph(G.v, fLenghtG.v) - }; - } else { - VectorF16<1, 32> lenght = LengthNoShuffle(A, E, C, G); - constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; - __m512h one = _mm512_loadu_ph(oneArr); - VectorF16<32, 1> fLenght(_mm512_div_ph(one, lenght.v)); - VectorF16<32, 1> fLenght2(lenght.v); - - VectorF16<32, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,1,1,1,1,8,8,8,8,9,9,9,9,16,16,16,16,17,17,17,17,24,24,24,24,25,25,25,25}}>(); - VectorF16<32, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,3,3,3,3,10,10,10,10,11,11,11,11,18,18,18,18,19,19,19,19,26,26,26,26,27,27,27,27}}>(); - VectorF16<32, 1> fLenghtE = fLenght.template Shuffle<{{4,4,4,4,5,5,5,5,12,12,12,12,13,13,13,13,20,20,20,20,21,21,21,21,28,28,28,28,29,29,29,29}}>(); - VectorF16<32, 1> fLenghtG = fLenght.template Shuffle<{{6,6,6,6,7,7,7,7,14,14,14,14,15,15,15,15,22,22,22,22,23,23,23,23,30,30,30,30,31,31,31,31}}>(); - - return { - VectorF16(_mm512_mul_ph(A.v, fLenghtA.v)), - VectorF16(_mm512_mul_ph(C.v, fLenghtC.v)), - VectorF16(_mm512_mul_ph(E.v, fLenghtE.v)), - VectorF16(_mm512_mul_ph(G.v, fLenghtG.v)), - }; - } - } - - constexpr static std::array, VectorBase::BatchSize> NormalizePack( - VectorF16 A, - VectorF16 E - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - VectorF16<1, 8> lenght = LengthNoShuffle(A, E); - constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1}; - __m128h one = _mm_loadu_ph(oneArr); - VectorF16<8, 1> fLenght(_mm_div_ph(one, lenght.v)); - - VectorF16<8, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,2,2,3,3}}>(); - VectorF16<8, 1> fLenghtE = fLenght.template Shuffle<{{4,4,5,5,6,6,7,7}}>(); - - return { - _mm_mul_ph(A.v, fLenghtA.v), - _mm_mul_ph(E.v, fLenghtE.v), - }; - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - VectorF16<1, 16> lenght = LengthNoShuffle(A, E); - constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; - __m256h one = _mm256_loadu_ph(oneArr); - VectorF16<16, 1> fLenght(_mm256_div_ph(one, lenght.v)); - - VectorF16<16, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11}}>(); - VectorF16<16, 1> fLenghtE = fLenght.template Shuffle<{{4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15}}>(); - - return { - _mm256_mul_ph(A.v, fLenghtA.v), - _mm256_mul_ph(E.v, fLenghtE.v), - }; - } else { - VectorF16<1, 32> lenght = LengthNoShuffle(A, E); - constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; - __m512h one = _mm512_loadu_ph(oneArr); - VectorF16<32, 1> fLenght(_mm512_div_ph(one, lenght.v)); - - VectorF16<32, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11,16,16,17,17,18,18,19,19,24,24,25,25,26,26,27,27}}>(); - VectorF16<32, 1> fLenghtE = fLenght.template Shuffle<{{4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15,20,20,21,21,22,22,23,23,28,28,29,29,30,30,31,31}}>(); - - return { - _mm512_mul_ph(A.v, fLenghtA.v), - _mm512_mul_ph(E.v, fLenghtE.v), - }; - } - } - - constexpr static VectorF16<1, Packing*4> LengthPack( - VectorF16 A, - VectorF16 C, - VectorF16 E, - VectorF16 G - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - VectorF16<1, Packing*4> lenghtSq = LengthSqPack(A, C, E, G); - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16<1, Packing*4>(_mm_sqrt_ph(lenghtSq.v)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16<1, Packing*4>(_mm256_sqrt_ph(lenghtSq.v)); - } else { - return VectorF16<1, Packing*4>(_mm512_sqrt_ph(lenghtSq.v)); - } - } - - constexpr static VectorF16<1, Packing*2> LengthPack( - VectorF16 A, - VectorF16 E - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - VectorF16<1, Packing*2> lenghtSq = LengthSqPack(A, E); - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16<1, Packing*2>(_mm_sqrt_ph(lenghtSq.v)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16<1, Packing*2>(_mm256_sqrt_ph(lenghtSq.v)); - } else { - return VectorF16<1, Packing*2>(_mm512_sqrt_ph(lenghtSq.v)); - } - } - - constexpr static VectorF16<1, Packing*4> LengthSqPack( - VectorF16 A, - VectorF16 C, - VectorF16 E, - VectorF16 G - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - return DotPack(A, A, C, C, E, E, G, G); - } - - constexpr static VectorF16<1, Packing*2> LengthSqPack( - VectorF16 A, - VectorF16 E - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - return DotPack(A, A, E, E); - } - - constexpr static VectorF16<1, Packing*4> DotPack( - VectorF16 A0, VectorF16 A1, - VectorF16 C0, VectorF16 C1, - VectorF16 E0, VectorF16 E1, - VectorF16 G0, VectorF16 G1 - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - return DotNoShuffle(A0, A1, E0, E1, C0, C1, G0, G1); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - VectorF16<16, 1> vec(DotNoShuffle(A0, A1, C0, C1, E0, E1, G0, G1).v); - vec = vec.template Shuffle<{{ - 0,1,8,9, - 4,5,12,13, - 2,3,10,11, - 6,7,14,15 - }}>(); - return vec.v; - } else { - VectorF16<32, 1> vec(DotNoShuffle(A0, A1, C0, C1, E0, E1, G0, G1).v); - vec = vec.template Shuffle<{{ - 0,1,8,9, - 16,17,24,25, - - 4,5,12,13, - 20,21,28,29, - - 2,3,10,11, - 18,19,24,25, - - 6,7,14,15, - 22,23,30,31 - }}>(); - return vec.v; - } - } - - constexpr static VectorF16<1, Packing*2> DotPack( - VectorF16 A0, VectorF16 A1, - VectorF16 E0, VectorF16 E1 - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - return DotNoShuffle(A0, A1, E0, E1); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - VectorF16<16, 1> vec(DotNoShuffle(A0, A1, E0, E1).v); - vec = vec.template Shuffle<{{0,1,2,3,8,9,10,11,4,5,6,7,12,13,14,15}}>(); - return vec.v; - } else { - VectorF16<32, 1> vec(DotNoShuffle(A0, A1, E0, E1).v); - vec = vec.template Shuffle<{{0,1,2,3,8,9,10,11,16,17,18,19,24,25,26,27,4,5,6,7,12,13,14,15,20,21,22,23,28,29,30,31}}>(); - return vec.v; - } - } - - - private: - constexpr static VectorF16<1, Packing*4> LengthNoShuffle( - VectorF16 A, - VectorF16 C, - VectorF16 E, - VectorF16 G - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - VectorF16<1, Packing*4> lenghtSq = LengthSqNoShuffle(A, C, E, G); - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16<1, Packing*4>(_mm_sqrt_ph(lenghtSq.v)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16<1, Packing*4>(_mm256_sqrt_ph(lenghtSq.v)); - } else { - return VectorF16<1, Packing*4>(_mm512_sqrt_ph(lenghtSq.v)); - } - } - - constexpr static VectorF16<1, Packing*2> LengthNoShuffle( - VectorF16 A, - VectorF16 E - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - VectorF16<1, Packing*2> lenghtSq = LengthSqNoShuffle(A, E); - if constexpr(std::is_same_v::VectorType, __m128h>) { - return VectorF16<1, Packing*2>(_mm_sqrt_ph(lenghtSq.v)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - return VectorF16<1, Packing*2>(_mm256_sqrt_ph(lenghtSq.v)); - } else { - return VectorF16<1, Packing*2>(_mm512_sqrt_ph(lenghtSq.v)); - } - } - - constexpr static VectorF16<1, Packing*4> LengthSqNoShuffle( - VectorF16 A, - VectorF16 C, - VectorF16 E, - VectorF16 G - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - return DotNoShuffle(A, A, C, C, E, E, G, G); - } - - constexpr static VectorF16<1, Packing*2> LengthSqNoShuffle( - VectorF16 A, - VectorF16 E - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - return DotNoShuffle(A, A, E, E); - } - - - constexpr static VectorF16<1, Packing*4> DotNoShuffle( - VectorF16 A0, VectorF16 A1, - VectorF16 C0, VectorF16 C1, - VectorF16 E0, VectorF16 E1, - VectorF16 G0, VectorF16 G1 - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - __m128h mulA = _mm_mul_ph(A0.v, A1.v); - __m128h mulC = _mm_mul_ph(C0.v, C1.v); - - __m128i row12Temp1 = _mm_unpacklo_epi16(_mm_castph_si128(mulA), _mm_castph_si128(mulC)); // A1 C1 A2 C2 A3 C3 A4 C4 - __m128i row34Temp1 = _mm_unpackhi_epi16(_mm_castph_si128(mulA), _mm_castph_si128(mulC)); // B1 D1 B2 D2 B3 D3 B4 D4 - __m128i row1TempTemp1 = row12Temp1; - - __m128h mulE = _mm_mul_ph(E0.v, E1.v); - __m128h mulG = _mm_mul_ph(G0.v, G1.v); - - __m128i row12Temp2 = _mm_unpacklo_epi16(_mm_castph_si128(mulE), _mm_castph_si128(mulG)); // E1 G1 E2 G2 E3 G3 E4 G4 - __m128i row12Temp2Temp = row12Temp2; - __m128i row34Temp2 = _mm_unpackhi_epi16(_mm_castph_si128(mulE), _mm_castph_si128(mulG)); // F1 H1 F2 H2 F3 H3 F4 H4 - - row12Temp1 = _mm_unpacklo_epi16(row12Temp1, row12Temp2); // A1 E1 C1 G1 A2 E2 C2 G2 - row12Temp2 = _mm_unpacklo_epi16(row34Temp1, row34Temp2); // B1 F1 D1 H1 B2 F2 D2 H2 - row34Temp2 = _mm_unpackhi_epi16(row34Temp1, row34Temp2); // B3 F3 D3 H3 B4 F4 D4 H4 - row34Temp1 = _mm_unpackhi_epi16(row1TempTemp1, row12Temp2Temp); // A3 E3 C3 G3 A4 E4 C4 G4 - - __m128h row1 = _mm_castsi128_ph(_mm_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 E1 F1 C1 D1 G1 H1 - __m128h row2 = _mm_castsi128_ph(_mm_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 E2 F2 C2 D2 G2 H2 - __m128h row3 = _mm_castsi128_ph(_mm_unpacklo_epi16(row34Temp1, row34Temp2));// A3 B3 E3 F3 C3 D3 G3 H3 - __m128h row4 = _mm_castsi128_ph(_mm_unpackhi_epi16(row34Temp1, row34Temp2));// A4 B4 E4 F4 C4 D4 G4 H4 - - row1 = _mm_add_ph(row1, row2); - row1 = _mm_add_ph(row1, row3); - row1 = _mm_add_ph(row1, row4); - - return row1; - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - __m256h mulA = _mm256_mul_ph(A0.v, A1.v); - __m256h mulC = _mm256_mul_ph(C0.v, C1.v); - - __m256i row12Temp1 = _mm256_unpacklo_epi16(_mm256_castph_si256(mulA), _mm256_castph_si256(mulC)); // A1 C1 A2 C2 A3 C3 A4 C4 - __m256i row34Temp1 = _mm256_unpackhi_epi16(_mm256_castph_si256(mulA), _mm256_castph_si256(mulC)); // B1 D1 B2 D2 B3 D3 B4 D4 - __m256i row1TempTemp1 = row12Temp1; - - __m256h mulE = _mm256_mul_ph(E0.v, E1.v); - __m256h mulG = _mm256_mul_ph(G0.v, G1.v); - - __m256i row12Temp2 = _mm256_unpacklo_epi16(_mm256_castph_si256(mulE), _mm256_castph_si256(mulG)); // E1 G1 E2 G2 E3 G3 E4 G4 - __m256i row12Temp2Temp = row12Temp2; - __m256i row34Temp2 = _mm256_unpackhi_epi16(_mm256_castph_si256(mulE), _mm256_castph_si256(mulG)); // F1 H1 F2 H2 F3 H3 F4 H4 - - row12Temp1 = _mm256_unpacklo_epi16(row12Temp1, row12Temp2); // A1 E1 C1 G1 A2 E2 C2 G2 - row12Temp2 = _mm256_unpacklo_epi16(row34Temp1, row34Temp2); // B1 F1 D1 H1 B2 F2 D2 H2 - row34Temp2 = _mm256_unpackhi_epi16(row34Temp1, row34Temp2); // B3 F3 D3 H3 B4 F4 D4 H4 - row34Temp1 = _mm256_unpackhi_epi16(row1TempTemp1, row12Temp2Temp); // A3 E3 C3 G3 A4 E4 C4 G4 - - __m256h row1 = _mm256_castsi256_ph(_mm256_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 E1 F1 C1 D1 G1 H1 - __m256h row2 = _mm256_castsi256_ph(_mm256_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 E2 F2 C2 D2 G2 H2 - __m256h row3 = _mm256_castsi256_ph(_mm256_unpacklo_epi16(row34Temp1, row34Temp2));// A3 B3 E3 F3 C3 D3 G3 H3 - __m256h row4 = _mm256_castsi256_ph(_mm256_unpackhi_epi16(row34Temp1, row34Temp2));// A4 B4 E4 F4 C4 D4 G4 H4 - - row1 = _mm256_add_ph(row1, row2); - row1 = _mm256_add_ph(row1, row3); - row1 = _mm256_add_ph(row1, row4); - - return row1; - } else { - __m512h mulA = _mm512_mul_ph(A0.v, A1.v); - __m512h mulC = _mm512_mul_ph(C0.v, C1.v); - - __m512i row12Temp1 = _mm512_unpacklo_epi16(_mm512_castph_si512(mulA), _mm512_castph_si512(mulC)); // A1 C1 A2 C2 A3 C3 A4 C4 - __m512i row34Temp1 = _mm512_unpackhi_epi16(_mm512_castph_si512(mulA), _mm512_castph_si512(mulC)); // B1 D1 B2 D2 B3 D3 B4 D4 - __m512i row1TempTemp1 = row12Temp1; - - __m512h mulE = _mm512_mul_ph(E0.v, E1.v); - __m512h mulG = _mm512_mul_ph(G0.v, G1.v); - - __m512i row12Temp2 = _mm512_unpacklo_epi16(_mm512_castph_si512(mulE), _mm512_castph_si512(mulG)); // E1 G1 E2 G2 E3 G3 E4 G4 - __m512i row12Temp2Temp = row12Temp2; - __m512i row34Temp2 = _mm512_unpackhi_epi16(_mm512_castph_si512(mulE), _mm512_castph_si512(mulG)); // F1 H1 F2 H2 F3 H3 F4 H4 - - row12Temp1 = _mm512_unpacklo_epi16(row12Temp1, row12Temp2); // A1 E1 C1 G1 A2 E2 C2 G2 - row12Temp2 = _mm512_unpacklo_epi16(row34Temp1, row34Temp2); // B1 F1 D1 H1 B2 F2 D2 H2 - row34Temp2 = _mm512_unpackhi_epi16(row34Temp1, row34Temp2); // B3 F3 D3 H3 B4 F4 D4 H4 - row34Temp1 = _mm512_unpackhi_epi16(row1TempTemp1, row12Temp2Temp); // A3 E3 C3 G3 A4 E4 C4 G4 - - __m512h row1 = _mm512_castsi512_ph(_mm512_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 E1 F1 C1 D1 G1 H1 - __m512h row2 = _mm512_castsi512_ph(_mm512_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 E2 F2 C2 D2 G2 H2 - __m512h row3 = _mm512_castsi512_ph(_mm512_unpacklo_epi16(row34Temp1, row34Temp2));// A3 B3 E3 F3 C3 D3 G3 H3 - __m512h row4 = _mm512_castsi512_ph(_mm512_unpackhi_epi16(row34Temp1, row34Temp2));// A4 B4 E4 F4 C4 D4 G4 H4 - - row1 = _mm512_add_ph(row1, row2); - row1 = _mm512_add_ph(row1, row3); - row1 = _mm512_add_ph(row1, row4); - - return row1; - } - } - - constexpr static VectorF16<1, Packing*2> DotNoShuffle( - VectorF16 A0, VectorF16 A1, - VectorF16 E0, VectorF16 E1 - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - if constexpr(std::is_same_v::VectorType, __m128h>) { - __m128h mulA = _mm_mul_ph(A0.v, A1.v); - __m128h mulE = _mm_mul_ph(E0.v, E1.v); - __m128i row12Temp1 = _mm_unpacklo_epi16(_mm_castph_si128(mulA), _mm_castph_si128(mulE)); // A1 E1 A2 E2 B1 F1 B2 F2 - __m128i row12Temp2 = _mm_unpackhi_epi16(_mm_castph_si128(mulA), _mm_castph_si128(mulE)); // C1 G1 C2 G2 D1 H1 D2 H2 - __m128i row12Temp1Temp = row12Temp1; - - row12Temp1 = _mm_unpacklo_epi16(row12Temp1, row12Temp2); // A1 C1 E1 G1 A2 C2 E2 G2 - row12Temp2 = _mm_unpackhi_epi16(row12Temp1Temp, row12Temp2); // B1 D1 F1 H1 B2 D2 F2 H2 - - __m128h row1 = _mm_castsi128_ph(_mm_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 C1 D1 E1 F1 G1 H1 - __m128h row2 = _mm_castsi128_ph(_mm_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 C2 D2 E2 F2 G2 H2 - - return _mm_add_ph(row1, row2); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - __m256h mulA = _mm256_mul_ph(A0.v, A1.v); - __m256h mulE = _mm256_mul_ph(E0.v, E1.v); - - __m256i row12Temp1 = _mm256_unpacklo_epi16(_mm256_castph_si256(mulA), _mm256_castph_si256(mulE)); // A1 E1 A2 E2 B1 F1 B2 F2 - - __m256i row12Temp2 = _mm256_unpackhi_epi16(_mm256_castph_si256(mulA), _mm256_castph_si256(mulE)); // C1 G1 C2 G2 D1 H1 D2 H2 - __m256i row12Temp1Temp = row12Temp1; - - row12Temp1 = _mm256_unpacklo_epi16(row12Temp1, row12Temp2); // A1 C1 E1 G1 A2 C2 E2 G2 - row12Temp2 = _mm256_unpackhi_epi16(row12Temp1Temp, row12Temp2); // B1 D1 F1 H1 B2 D2 F2 H2 - - __m256h row1 = _mm256_castsi256_ph(_mm256_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 C1 D1 E1 F1 G1 H1 - __m256h row2 = _mm256_castsi256_ph(_mm256_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 C2 D2 E2 F2 G2 H2 - __m256h result = _mm256_add_ph(row1, row2); - - return result; - } else { - __m512h mulA = _mm512_mul_ph(A0.v, A1.v); - __m512h mulE = _mm512_mul_ph(E0.v, E1.v); - __m512i row12Temp1 = _mm512_unpacklo_epi16(_mm512_castph_si512(mulA), _mm512_castph_si512(mulE)); // A1 E1 A2 E2 B1 F1 B2 F2 - __m512i row12Temp2 = _mm512_unpackhi_epi16(_mm512_castph_si512(mulA), _mm512_castph_si512(mulE)); // C1 G1 C2 G2 D1 H1 D2 H2 - __m512i row12Temp1Temp = row12Temp1; - - row12Temp1 = _mm512_unpacklo_epi16(row12Temp1, row12Temp2); // A1 C1 E1 G1 A2 C2 E2 G2 - row12Temp2 = _mm512_unpackhi_epi16(row12Temp1Temp, row12Temp2); // B1 D1 F1 H1 B2 D2 F2 H2 - - __m512h row1 = _mm512_castsi512_ph(_mm512_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 C1 D1 E1 F1 G1 H1 - __m512h row2 = _mm512_castsi512_ph(_mm512_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 C2 D2 E2 F2 G2 H2 - __m512h result = _mm512_add_ph(row1, row2); - - return result; - } - } - public: - - template ShuffleValues> - constexpr static VectorF16 Blend(VectorF16 a, VectorF16 b) { - constexpr auto mask = VectorBase::template GetBlendMaskEpi16(); - if constexpr(std::is_same_v::VectorType, __m128h>) { - return _mm_castsi128_ph(_mm_blend_epi16(_mm_castph_si128(a.v), _mm_castph_si128(b.v), mask)); - } else if constexpr(std::is_same_v::VectorType, __m256h>) { - #ifndef __AVX512BW__ - #ifndef __AVX512VL__ - static_assert(false, "No __AVX512BW__ and __AVX512VL__ support"); - #endif - #endif - return _mm256_castsi256_ph(_mm256_mask_blend_epi16(mask, _mm256_castph_si256(a.v), _mm256_castph_si256(b.v))); - } else { - return _mm512_castsi512_ph(_mm512_mask_blend_epi16(mask, _mm512_castph_si512(a.v), _mm512_castph_si512(b.v))); - } - } - - constexpr static VectorF16 Rotate(VectorF16<3, Packing> v, VectorF16<4, Packing> q) requires(Len == 3) { - VectorF16<3, Packing> qv(q); - VectorF16 t = Cross(qv, v) * _Float16(2); - return v + t * q.template Shuffle<{{3,3,3,3}}>() + Cross(qv, t); - } - - constexpr static VectorF16<4, 2> RotatePivot(VectorF16<3, Packing> v, VectorF16<4, Packing> q, VectorF16<3, Packing> pivot) requires(Len == 3) { - VectorF16 translated = v - pivot; - VectorF16<3, Packing> qv(q.v); - VectorF16 t = Cross(qv, translated) * _Float16(2); - VectorF16 rotated = translated + t * q.template Shuffle<{{3,3,3,3}}>() + Cross(qv, t); - return rotated + pivot; - } - - constexpr static VectorF16<4, Packing> QuanternionFromEuler(VectorF16<3, Packing> EulerHalf) requires(Len == 4) { - std::tuple, VectorF16<3, Packing>> sinCos = EulerHalf.SinCos(); - VectorF16<4, Packing> sin = std::get<0>(sinCos); - VectorF16<4, Packing> cos = std::get<1>(sinCos); - - VectorF16<4, Packing> row1 = cos.template Shuffle<{{0,0,0,0}}>(); - row1 = Blend<{{0,1,1,1}}>(sin, row1); - - VectorF16<4, Packing> row2 = cos.template Shuffle<{{1,1,1,1}}>(); - row2 = Blend<{{1,0,1,1}}>(sin, row2); - - row1 *= row2; - - VectorF16<4, Packing> row3 = cos.template Shuffle<{{2,2,2,2}}>(); - row3 = Blend<{{1,1,0,1}}>(sin, row3); - - row1 *= row3; - - VectorF16<4, Packing> row4 = sin.template Shuffle<{{0,0,0,0}}>(); - row4 = Blend<{{0,1,1,1}}>(cos, row4); - - VectorF16<4, Packing> row5 = sin.template Shuffle<{{1,1,1,1}}>(); - row5 = Blend<{{1,0,1,1}}>(cos, row5); - - row4 *= row5; - - VectorF16<4, Packing> row6 = sin.template Shuffle<{{2,2,2,2}}>(); - row6 = Blend<{{1,1,0,1}}>(cos, row6); - row6 = row6.template Negate<{{true,false,true,false}}>(); - - row1 = MulitplyAdd(row4, row6, row1); - - return row1; - } - }; -} - - -#elif !defined(__x86_64) && defined(__FLT16_MAX__) -// Scalar software fallback for non-x86_64 targets that still have _Float16. -namespace Crafter { - export template - struct VectorF16 : public VectorBase { - template - friend struct VectorF16; - using Base = VectorBase; - static constexpr std::uint8_t NElems = Base::AlignmentElement; - - constexpr VectorF16() = default; - constexpr VectorF16(typename Base::VectorType vv) { - this->v = vv; - } - constexpr VectorF16(const _Float16* vB) { Load(vB); } - constexpr VectorF16(_Float16 val) { - for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = val; - } - - constexpr void Load(const _Float16* vB) { - for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = vB[i]; - } - constexpr void Store(_Float16* vB) const { - for (std::uint8_t i = 0; i < NElems; ++i) vB[i] = this->v[i]; - } - - template - constexpr std::array<_Float16, NElems> Store() const { - std::array<_Float16, NElems> r{}; - Store(r.data()); - return r; - } - - template - constexpr operator VectorF16() const { - VectorF16 r; - const std::uint8_t copyLen = (BLen < Len) ? BLen : Len; - const std::uint8_t copyPack = (BPacking < Packing) ? BPacking : Packing; - for (std::uint8_t p = 0; p < copyPack; ++p) - for (std::uint8_t i = 0; i < copyLen; ++i) - r.v[p * BLen + i] = this->v[p * Len + i]; - return r; - } - - constexpr VectorF16 operator+(VectorF16 b) const { - VectorF16 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] + b.v[i]; - return r; - } - constexpr VectorF16 operator-(VectorF16 b) const { - VectorF16 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] - b.v[i]; - return r; - } - constexpr VectorF16 operator*(VectorF16 b) const { - VectorF16 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] * b.v[i]; - return r; - } - constexpr VectorF16 operator/(VectorF16 b) const { - VectorF16 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] / b.v[i]; - return r; - } - - constexpr void operator+=(VectorF16 b) { for (std::uint8_t i=0;iv[i] += b.v[i]; } - constexpr void operator-=(VectorF16 b) { for (std::uint8_t i=0;iv[i] -= b.v[i]; } - constexpr void operator*=(VectorF16 b) { for (std::uint8_t i=0;iv[i] *= b.v[i]; } - constexpr void operator/=(VectorF16 b) { for (std::uint8_t i=0;iv[i] /= b.v[i]; } - - constexpr VectorF16 operator+(_Float16 b) const { return *this + VectorF16(b); } - constexpr VectorF16 operator-(_Float16 b) const { return *this - VectorF16(b); } - constexpr VectorF16 operator*(_Float16 b) const { return *this * VectorF16(b); } - constexpr VectorF16 operator/(_Float16 b) const { return *this / VectorF16(b); } - constexpr void operator+=(_Float16 b) { *this += VectorF16(b); } - constexpr void operator-=(_Float16 b) { *this -= VectorF16(b); } - constexpr void operator*=(_Float16 b) { *this *= VectorF16(b); } - constexpr void operator/=(_Float16 b) { *this /= VectorF16(b); } - - constexpr VectorF16 operator-() const { - VectorF16 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = -this->v[i]; - return r; - } - - constexpr bool operator==(VectorF16 b) const { - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - if (this->v[p * Len + i] != b.v[p * Len + i]) return false; - return true; - } - constexpr bool operator!=(VectorF16 b) const { return !(*this == b); } - - template - constexpr VectorF16 ExtractLo() const { - VectorF16 r; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < ExtractLen; ++i) - r.v[p * ExtractLen + i] = this->v[p * Len + i]; - return r; - } - - // Transcendentals are computed via float since libstdc++ doesn't always - // provide overloads for _Float16; the result is rounded back to half. - constexpr VectorF16 Cos() const { - VectorF16 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = static_cast<_Float16>(std::cos(static_cast(this->v[i]))); - return r; - } - constexpr VectorF16 Sin() const { - VectorF16 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = static_cast<_Float16>(std::sin(static_cast(this->v[i]))); - return r; - } - constexpr std::tuple, VectorF16> SinCos() const { - return { Sin(), Cos() }; - } - - template values> - constexpr VectorF16 Negate() const { - VectorF16 r; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - r.v[p * Len + i] = values[i] ? -this->v[p * Len + i] : this->v[p * Len + i]; - return r; - } - - static constexpr VectorF16 MulitplyAdd(VectorF16 a, VectorF16 b, VectorF16 add) { - VectorF16 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = a.v[i] * b.v[i] + add.v[i]; - return r; - } - static constexpr VectorF16 MulitplySub(VectorF16 a, VectorF16 b, VectorF16 sub) { - VectorF16 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = a.v[i] * b.v[i] - sub.v[i]; - return r; - } - - constexpr static VectorF16 Cross(VectorF16 a, VectorF16 b) requires(Len == 3) { - VectorF16 r; - for (std::uint8_t p = 0; p < Packing; ++p) { - const std::uint8_t base = p * 3; - r.v[base + 0] = a.v[base + 1] * b.v[base + 2] - a.v[base + 2] * b.v[base + 1]; - r.v[base + 1] = a.v[base + 2] * b.v[base + 0] - a.v[base + 0] * b.v[base + 2]; - r.v[base + 2] = a.v[base + 0] * b.v[base + 1] - a.v[base + 1] * b.v[base + 0]; - } - return r; - } - - template ShuffleValues> - constexpr VectorF16 Shuffle() const { - VectorF16 r; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - r.v[p * Len + i] = this->v[p * Len + ShuffleValues[i]]; - return r; - } - - template ShuffleValues> - constexpr static VectorF16 Blend(VectorF16 a, VectorF16 b) { - VectorF16 r; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - r.v[p * Len + i] = ShuffleValues[i] ? b.v[p * Len + i] : a.v[p * Len + i]; - return r; - } - - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto LengthSq(VectorF16 first, Rest... rest) { - constexpr std::uint8_t N = VectorBase::BatchSize; - VectorF16<1, static_cast(Packing * N)> r; - std::array, N> args{ first, rest... }; - for (std::uint8_t i = 0; i < N; ++i) - for (std::uint8_t p = 0; p < Packing; ++p) { - _Float16 acc = _Float16(0); - for (std::uint8_t k = 0; k < Len; ++k) { - _Float16 x = args[i].v[p * Len + k]; - acc += x * x; - } - r.v[i * Packing + p] = acc; - } - return r; - } - - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto Length(VectorF16 first, Rest... rest) { - auto sq = LengthSq(first, rest...); - for (std::uint8_t i = 0; i < decltype(sq)::NElems; ++i) - sq.v[i] = static_cast<_Float16>(std::sqrt(static_cast(sq.v[i]))); - return sq; - } - - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto Normalize(VectorF16 first, Rest... rest) { - auto normOne = [](VectorF16 u) { - VectorF16 out; - for (std::uint8_t p = 0; p < Packing; ++p) { - float acc = 0.0f; - for (std::uint8_t k = 0; k < Len; ++k) { - float x = static_cast(u.v[p * Len + k]); - acc += x * x; - } - _Float16 invLen = acc > 0.0f ? static_cast<_Float16>(1.0f / std::sqrt(acc)) : _Float16(0); - for (std::uint8_t k = 0; k < Len; ++k) - out.v[p * Len + k] = u.v[p * Len + k] * invLen; - } - return out; - }; - return std::array, VectorBase::BatchSize>{ normOne(first), normOne(rest)... }; - } - - constexpr static VectorF16 Rotate(VectorF16<3, Packing> v, VectorF16<4, Packing> q) requires(Len == 3) { - VectorF16<3, Packing> qv; - VectorF16<3, Packing> qwBroadcast; - for (std::uint8_t p = 0; p < Packing; ++p) { - qv.v[p * 3 + 0] = q.v[p * 4 + 0]; - qv.v[p * 3 + 1] = q.v[p * 4 + 1]; - qv.v[p * 3 + 2] = q.v[p * 4 + 2]; - for (std::uint8_t i = 0; i < 3; ++i) qwBroadcast.v[p * 3 + i] = q.v[p * 4 + 3]; - } - VectorF16<3, Packing> t = Cross(qv, v) * _Float16(2); - return v + t * qwBroadcast + Cross(qv, t); - } - - constexpr static VectorF16<3, Packing> RotatePivot(VectorF16<3, Packing> v, VectorF16<4, Packing> q, VectorF16<3, Packing> pivot) requires(Len == 3) { - VectorF16<3, Packing> translated = v - pivot; - return Rotate(translated, q) + pivot; - } - - constexpr static VectorF16<4, Packing> QuanternionFromEuler(VectorF16<3, Packing> eulerHalf) requires(Len == 4) { - VectorF16<4, Packing> r; - for (std::uint8_t p = 0; p < Packing; ++p) { - float roll = static_cast(eulerHalf.v[p * 3 + 0]); - float pitch = static_cast(eulerHalf.v[p * 3 + 1]); - float yaw = static_cast(eulerHalf.v[p * 3 + 2]); - float sr = std::sin(roll), cr = std::cos(roll); - float sp = std::sin(pitch), cp = std::cos(pitch); - float sy = std::sin(yaw), cy = std::cos(yaw); - r.v[p * 4 + 0] = static_cast<_Float16>(sr * cp * cy - cr * sp * sy); - r.v[p * 4 + 1] = static_cast<_Float16>(cr * sp * cy + sr * cp * sy); - r.v[p * 4 + 2] = static_cast<_Float16>(cr * cp * sy - sr * sp * cy); - r.v[p * 4 + 3] = static_cast<_Float16>(cr * cp * cy + sr * sp * sy); - } - return r; - } - }; -} -#endif - -#if defined(__FLT16_MAX__) && (!defined(__x86_64) || defined(__AVX512FP16__)) -export template -struct std::formatter> : std::formatter { - constexpr auto format(const Crafter::VectorF16& obj, format_context& ctx) const { - std::array<_Float16, Crafter::VectorF16::AlignmentElement> vec = obj.template Store(); - std::string out = "{"; - for(std::uint32_t i = 0; i < Packing; i++) { - out += "{"; - for(std::uint32_t i2 = 0; i2 < Len; i2++) { - out += std::format("{}", static_cast(vec[i * Len + i2])); - if (i2 + 1 < Len) out += ","; - } - out += "}"; - } - out += "}"; - return std::formatter::format(out, ctx); - } -}; +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +module; +#ifdef __x86_64 +#include +#endif +export module Crafter.Math:VectorF16; +import std; +import :Common; + +// A real VectorF16 struct is provided only when _Float16 is available AND we +// are either on an x86_64 build with AVX512FP16 (intrinsic impl below) or on a +// non-x86_64 target (scalar fallback further below). On x86_64 without +// AVX512FP16 or anywhere without _Float16, Crafter.Math:Basic aliases +// VectorF16 to VectorF32 instead. +#ifdef __AVX512FP16__ +namespace Crafter { + export template + struct VectorF16 : public VectorBase { + template + friend struct VectorF16; + + constexpr VectorF16() = default; + constexpr VectorF16(VectorBase::VectorType v) { + this->v = v; + } + constexpr VectorF16(const _Float16* vB) { + Load(vB); + }; + constexpr VectorF16(_Float16 val) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + this->v = _mm_set1_ph(val); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + this->v = _mm256_set1_ph(val); + } else { + this->v = _mm512_set1_ph(val); + } + }; + constexpr void Load(const _Float16* vB) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + this->v = _mm_loadu_ph(vB); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + this->v = _mm256_loadu_ph(vB); + } else { + this->v = _mm512_loadu_ph(vB); + } + } + constexpr void Store(_Float16* vB) const { + if constexpr(std::is_same_v::VectorType, __m128h>) { + _mm_storeu_ph(vB, this->v); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + _mm256_storeu_ph(vB, this->v); + } else { + _mm512_storeu_ph(vB, this->v); + } + } + + template + constexpr std::array<_Float16, VectorBase::AlignmentElement> Store() const { + std::array<_Float16, VectorBase::AlignmentElement> returnArray; + Store(returnArray.data()); + return returnArray; + } + + template + constexpr operator VectorF16() const { + if constexpr (Len == BLen) { + if constexpr(std::is_same_v::VectorType, __m256h> && std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm256_castph256_ph128(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m512h> && std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm512_castph512_ph128(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m512h> && std::is_same_v::VectorType, __m256h>) { + return VectorF16(_mm512_castph512_ph256(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m128h> && std::is_same_v::VectorType, __m256h>) { + return VectorF16(_mm256_castph128_ph256(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m128h> && std::is_same_v::VectorType, __m512h>) { + return VectorF16(_mm512_castph128_ph512(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m256h> && std::is_same_v::VectorType, __m512h>) { + return VectorF16(_mm512_castph256_ph512(this->v)); + } else { + return VectorF16(this->v); + } + } else if constexpr (BLen <= Len) { + return this->template ExtractLo(); + } else { + if constexpr(std::is_same_v::VectorType, __m128h>) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + constexpr std::array::Alignment> shuffleMask = VectorBase::template GetExtractLoMaskEpi8(); + __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); + return VectorF16(_mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(this->v), shuffleVec))); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + constexpr std::array::AlignmentElement> permMask =VectorBase::template GetExtractLoMaskEpi16(); + __m256i permIdx = _mm256_loadu_epi16(permMask.data()); + __m256i result = _mm256_permutexvar_epi16(permIdx, _mm_castph_si256(this->v)); + return VectorF16(_mm_castsi128_ph(_mm256_castsi256_si128(result))); + } else { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); + __m512i permIdx = _mm512_loadu_epi16(permMask.data()); + __m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castph_si512(this->v)); + return VectorF16(_mm_castsi128_ph(_mm512_castsi512_si128(result))); + } + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); + __m256i permIdx = _mm256_loadu_epi16(permMask.data()); + __m256i result = _mm256_permutexvar_epi16(permIdx, _mm256_castsi128_si256(_mm_castph_si128(this->v))); + return VectorF16(_mm256_castsi256_ph(result)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); + __m256i permIdx = _mm256_loadu_epi16(permMask.data()); + __m256i result = _mm256_permutexvar_epi16(permIdx, _mm256_castph_si256(this->v)); + return VectorF16(_mm256_castsi256_ph(result)); + } else { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); + __m256i permIdx = _mm512_loadu_epi16(permMask.data()); + __m256i result = _mm512_permutexvar_epi16(permIdx, _mm512_castsi512_si256(_mm512_castph_si512(this->v))); + return VectorF16(_mm256_castsi256_ph(result)); + } + } else { + if constexpr(std::is_same_v::VectorType, __m128h>) { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); + __m512i permIdx = _mm512_loadu_epi16(permMask.data()); + __m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castsi128_si512(_mm_castph_si128(this->v))); + return VectorF16(_mm512_castsi512_ph(result)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); + __m512i permIdx = _mm512_loadu_epi16(permMask.data()); + __m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castsi256_si512(_mm256_castph_si256(this->v))); + return VectorF16(_mm512_castsi512_ph(result)); + } else { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); + __m512i permIdx = _mm512_loadu_epi16(permMask.data()); + __m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castph_si512(this->v)); + return VectorF16(_mm512_castsi512_ph(result)); + } + } + } + } + + constexpr VectorF16 operator+(VectorF16 b) const { + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm_add_ph(this->v, b.v)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16(_mm256_add_ph(this->v, b.v)); + } else { + return VectorF16(_mm512_add_ph(this->v, b.v)); + } + } + + constexpr VectorF16 operator-(VectorF16 b) const { + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm_sub_ph(this->v, b.v)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16(_mm256_sub_ph(this->v, b.v)); + } else { + return VectorF16(_mm512_sub_ph(this->v, b.v)); + } + } + + constexpr VectorF16 operator*(VectorF16 b) const { + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm_mul_ph(this->v, b.v)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16(_mm256_mul_ph(this->v, b.v)); + } else { + return VectorF16(_mm512_mul_ph(this->v, b.v)); + } + } + + constexpr VectorF16 operator/(VectorF16 b) const { + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm_div_ph(this->v, b.v)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16(_mm256_div_ph(this->v, b.v)); + } else { + return VectorF16(_mm512_div_ph(this->v, b.v)); + } + } + + + constexpr void operator+=(VectorF16 b) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + this->v = _mm_add_ph(this->v, b.v); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + this->v = _mm256_add_ph(this->v, b.v); + } else { + this->v = _mm512_add_ph(this->v, b.v); + } + } + + constexpr void operator-=(VectorF16 b) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + this->v = _mm_sub_ph(this->v, b.v); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + this->v = _mm256_sub_ph(this->v, b.v); + } else { + this->v = _mm512_sub_ph(this->v, b.v); + } + } + + constexpr void operator*=(VectorF16 b) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + this->v = _mm_mul_ph(this->v, b.v); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + this->v = _mm256_mul_ph(this->v, b.v); + } else { + this->v = _mm512_mul_ph(this->v, b.v); + } + } + + constexpr void operator/=(VectorF16 b) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + this->v = _mm_div_ph(this->v, b.v); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + this->v = _mm256_div_ph(this->v, b.v); + } else { + this->v = _mm512_div_ph(this->v, b.v); + } + } + + constexpr VectorF16 operator+(_Float16 b) { + VectorF16 vB(b); + return *this + vB; + } + + constexpr VectorF16 operator-(_Float16 b) { + VectorF16 vB(b); + return *this - vB; + } + + constexpr VectorF16 operator*(_Float16 b) { + VectorF16 vB(b); + return *this * vB; + } + + constexpr VectorF16 operator/(_Float16 b) { + VectorF16 vB(b); + return *this / vB; + } + + constexpr void operator+=(_Float16 b) { + VectorF16 vB(b); + *this += vB; + } + + constexpr void operator-=(_Float16 b) { + VectorF16 vB(b); + *this -= vB; + } + + constexpr void operator*=(_Float16 b) { + VectorF16 vB(b); + *this *= vB; + } + + constexpr void operator/=(_Float16 b) { + VectorF16 vB(b); + *this /= vB; + } + + constexpr VectorF16 operator-(){ + return Negate::GetAllTrue()>(); + } + + constexpr bool operator==(VectorF16 b) const { + if constexpr(std::is_same_v::VectorType, __m128h>) { + return _mm_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) == 255; + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return _mm256_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) == 65535; + } else { + return _mm512_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) == 4294967295; + } + } + + constexpr bool operator!=(VectorF16 b) const { + if constexpr(std::is_same_v::VectorType, __m128h>) { + return _mm_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) != 255; + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return _mm256_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) != 65535; + } else { + return _mm512_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) != 4294967295; + } + } + + template + constexpr VectorF16 ExtractLo() const { + if constexpr(Packing > 1) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + constexpr std::array::Alignment> shuffleMask = VectorBase::template GetExtractLoMaskEpi8(); + __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); + return VectorF16(_mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(this->v), shuffleVec))); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); + __m256i permIdx = _mm256_loadu_epi16(permMask.data()); + __m256i result = _mm256_permutexvar_epi16(permIdx, _mm256_castph_si256(this->v)); + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm256_castph256_ph128(_mm256_castsi256_ph(result))); + } else { + return VectorF16(_mm256_castsi256_ph(result)); + } + } else { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi16(); + __m512i permIdx = _mm512_loadu_epi16(permMask.data()); + __m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castph_si512(this->v)); + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm512_castph512_ph128(_mm512_castsi512_ph(result))); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16(_mm512_castph512_ph256(_mm512_castsi512_ph(result))); + } else { + return VectorF16(_mm512_castsi512_ph(result)); + } + } + } else { + if constexpr(std::is_same_v::VectorType, __m256h> && std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm256_castph256_ph128(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m512h> && std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm512_castph512_ph128(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m512h> && std::is_same_v::VectorType, __m256h>) { + return VectorF16(_mm512_castph512_ph256(this->v)); + } else { + return VectorF16(this->v); + } + } + } + + constexpr VectorF16 Cos() { + if constexpr (std::is_same_v::VectorType, __m128h>) { + __m256 wide = _mm256_cvtph_ps(_mm_castph_si128(this->v)); + wide = VectorBase::cos_f32x8(wide); + return VectorF16( + _mm_castsi128_ph(_mm256_cvtps_ph(wide, _MM_FROUND_TO_NEAREST_INT))); + + } else if constexpr (std::is_same_v::VectorType, __m256h>) { + __m512 wide = _mm512_cvtph_ps(_mm256_castph_si256(this->v)); + wide = VectorBase::cos_f32x16(wide); + return VectorF16( + _mm256_castsi256_ph(_mm512_cvtps_ph(wide, _MM_FROUND_TO_NEAREST_INT))); + + } else { + __m256i lo = _mm512_castsi512_si256(_mm512_castph_si512(this->v)); + __m256i hi = _mm512_extracti64x4_epi64(_mm512_castph_si512(this->v), 1); + __m512i cosLo =VectorBase::cos_f32x16(_mm512_cvtph_ps(lo)); + __m512i cosHi =VectorBase::cos_f32x16(_mm512_cvtph_ps(hi)); + __m256i lo_ph = _mm512_cvtps_ph(cosLo, _MM_FROUND_TO_NEAREST_INT); + __m256i hi_ph = _mm512_cvtps_ph(cosHi, _MM_FROUND_TO_NEAREST_INT); + return VectorF16(_mm512_castsi512_ph(_mm512_inserti64x4(_mm512_castsi256_si512(lo_ph), hi_ph, 1))); + } + } + + constexpr VectorF16 Sin() { + if constexpr (std::is_same_v::VectorType, __m128h>) { + __m256 wide = _mm256_cvtph_ps(_mm_castph_si128(this->v)); + wide = VectorBase::sin_f32x8(wide); + return VectorF16(_mm_castsi128_ph(_mm256_cvtps_ph(wide, _MM_FROUND_TO_NEAREST_INT))); + + } else if constexpr (std::is_same_v::VectorType, __m256h>) { + __m512 wide = _mm512_cvtph_ps(_mm256_castph_si256(this->v)); + wide = VectorBase::sin_f32x16(wide); + return VectorF16(_mm256_castsi256_ph(_mm512_cvtps_ph(wide, _MM_FROUND_TO_NEAREST_INT))); + + } else { + __m256i lo = _mm512_castsi512_si256(_mm512_castph_si512(this->v)); + __m256i hi = _mm512_extracti64x4_epi64(_mm512_castph_si512(this->v), 1); + __m512i loSin = VectorBase::sin_f32x16(_mm512_cvtph_ps(lo)); + __m512i hiSin = VectorBase::sin_f32x16(_mm512_cvtph_ps(hi)); + __m256i lo_ph = _mm512_cvtps_ph(loSin, _MM_FROUND_TO_NEAREST_INT); + __m256i hi_ph = _mm512_cvtps_ph(hiSin, _MM_FROUND_TO_NEAREST_INT); + return VectorF16(_mm512_castsi512_ph(_mm512_inserti64x4(_mm512_castsi256_si512(lo_ph), hi_ph, 1))); + } + } + + std::tuple, VectorF16> SinCos() { + if constexpr (std::is_same_v::VectorType, __m128h>) { + __m256 wide = _mm256_cvtph_ps(_mm_castph_si128(this->v)); + __m256 s, c; + VectorBase::sincos_f32x8(wide, s, c); + return { + VectorF16(_mm_castsi128_ph(_mm256_cvtps_ph(s, _MM_FROUND_TO_NEAREST_INT))), + VectorF16(_mm_castsi128_ph(_mm256_cvtps_ph(c, _MM_FROUND_TO_NEAREST_INT))) + }; + + } else if constexpr (std::is_same_v::VectorType, __m256h>) { + __m512 wide = _mm512_cvtph_ps(_mm256_castph_si256(this->v)); + __m512 s, c; + VectorBase::sincos_f32x16(wide, s, c); + return { + VectorF16(_mm256_castsi256_ph(_mm512_cvtps_ph(s, _MM_FROUND_TO_NEAREST_INT))), + VectorF16(_mm256_castsi256_ph(_mm512_cvtps_ph(c, _MM_FROUND_TO_NEAREST_INT))) + }; + + } else { + __m256i lo = _mm512_castsi512_si256(_mm512_castph_si512(this->v)); + __m256i hi = _mm512_extracti64x4_epi64(_mm512_castph_si512(this->v), 1); + + __m512 s_lo, c_lo, s_hi, c_hi; + VectorBase::sincos_f32x16(_mm512_cvtph_ps(lo), s_lo, c_lo); + VectorBase::sincos_f32x16(_mm512_cvtph_ps(hi), s_hi, c_hi); + + auto pack = [](__m256i lo_ph, __m256i hi_ph) { + return _mm512_castsi512_ph(_mm512_inserti64x4(_mm512_castsi256_si512(lo_ph), hi_ph, 1)); + }; + return { + VectorF16(pack(_mm512_cvtps_ph(s_lo, _MM_FROUND_TO_NEAREST_INT), _mm512_cvtps_ph(s_hi, _MM_FROUND_TO_NEAREST_INT))), + VectorF16(pack( _mm512_cvtps_ph(c_lo, _MM_FROUND_TO_NEAREST_INT), _mm512_cvtps_ph(c_hi, _MM_FROUND_TO_NEAREST_INT))) + }; + } + } + + template values> + constexpr VectorF16 Negate() { + std::array<_Float16, VectorBase::AlignmentElement> mask = VectorBase::template GetNegateMask(); + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm_castsi128_ph(_mm_xor_si128(_mm_castph_si128(this->v), _mm_loadu_epi16(mask.data())))); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16(_mm256_castsi256_ph(_mm256_xor_si256(_mm256_castph_si256(this->v), _mm256_loadu_epi16(mask.data())))); + } else { + return VectorF16(_mm512_castsi512_ph(_mm512_xor_si512(_mm512_castph_si512(this->v), _mm512_loadu_epi16(mask.data())))); + } + } + + static constexpr VectorF16 MulitplyAdd(VectorF16 a, VectorF16 b, VectorF16 add) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm_fmadd_ph(a.v, b.v, add.v)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16(_mm256_fmadd_ph(a.v, b.v, add.v)); + } else { + return VectorF16(_mm512_fmadd_ph(a.v, b.v, add.v)); + } + } + + static constexpr VectorF16 MulitplySub(VectorF16 a, VectorF16 b, VectorF16 sub) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm_fmsub_ph(a.v, b.v, sub.v)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16(_mm256_fmsub_ph(a.v, b.v, sub.v)); + } else { + return VectorF16(_mm512_fmsub_ph(a.v, b.v, sub.v)); + } + } + + constexpr static VectorF16 Cross(VectorF16 a, VectorF16 b) requires(Len == 3) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + constexpr std::array::AlignmentElement*2> shuffleMask1 = VectorBase::template GetShuffleMaskEpi8<{{1,2,0}}>(); + __m128i shuffleVec1 = _mm_loadu_epi8(shuffleMask1.data()); + __m128h row1 = _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(a.v), shuffleVec1)); + __m128h row4 = _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(b.v), shuffleVec1)); + + constexpr std::array::AlignmentElement*2> shuffleMask3 = VectorBase::template GetShuffleMaskEpi8<{{2,0,1}}>(); + __m128i shuffleVec3 = _mm_loadu_epi8(shuffleMask3.data()); + __m128h row3 = _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(a.v), shuffleVec3)); + __m128h row2 = _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(b.v), shuffleVec3)); + + __m128h result = _mm_mul_ph(row3, row4); + return _mm_fmsub_ph(row1,row2,result); + } else if constexpr (std::is_same_v::VectorType, __m256h>) { + constexpr std::array::AlignmentElement*2> shuffleMask1 = VectorBase::template GetShuffleMaskEpi8<{{1,2,0}}>(); + __m512i shuffleVec1 = _mm512_castsi256_si512(_mm256_loadu_epi8(shuffleMask1.data())); + __m256h row1 = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(a.v)), shuffleVec1))); + __m256h row4 = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(b.v)), shuffleVec1))); + + constexpr std::array::AlignmentElement*2> shuffleMask3 = VectorBase::template GetShuffleMaskEpi8<{{2,0,1}}>(); + + __m512i shuffleVec3 = _mm512_castsi256_si512(_mm256_loadu_epi8(shuffleMask3.data())); + __m256h row3 = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(a.v)), shuffleVec3))); + __m256h row2 = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(b.v)), shuffleVec3))); + + __m256h result = _mm256_mul_ph(row3, row4); + return _mm256_fmsub_ph(row1,row2,result); + } else { + constexpr std::array::AlignmentElement*2> shuffleMask1 = VectorBase::template GetShuffleMaskEpi8<{{1,2,0}}>(); + + __m512i shuffleVec1 = _mm512_loadu_epi8(shuffleMask1.data()); + __m512h row1 = _mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(a.v), shuffleVec1)); + __m512h row4 = _mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(b.v), shuffleVec1)); + + constexpr std::array::AlignmentElement*2> shuffleMask3 = VectorBase::template GetShuffleMaskEpi8<{{2,0,1}}>(); + + __m512i shuffleVec3 = _mm512_loadu_epi8(shuffleMask3.data()); + __m512h row3 = _mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(a.v), shuffleVec3)); + __m512h row2 = _mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(b.v), shuffleVec3)); + + __m512h result = _mm512_mul_ph(row3, row4); + return _mm512_fmsub_ph(row1,row2,result); + } + } + + constexpr static _Float16 Dot(VectorF16 a, VectorF16 b) requires(Packing == 1) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + __m128h mul = _mm_mul_ph(a.v, b.v); + return _mm_reduce_add_ph(mul); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + static_assert(std::is_same_v::VectorType>, "a.v is NOT VectorBase::VectorType"); + __m256h mul = _mm256_mul_ph(a.v, b.v); + return _mm256_reduce_add_ph(mul); + } else { + __m512h mul = _mm512_mul_ph(a.v, b.v); + return _mm512_reduce_add_ph(mul); + } + } + + template ShuffleValues> + constexpr VectorF16 Shuffle() { + if constexpr(VectorBase::template CheckEpi32Shuffle()) { + constexpr std::uint8_t imm = VectorBase::template GetShuffleMaskEpi32(); + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16(_mm_castsi128_ph(_mm_shuffle_epi32(_mm_castph_si128(this->v), imm))); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16(_mm256_castsi256_ph(_mm256_shuffle_epi32(_mm256_castph_si256(this->v), imm))); + } else { + return VectorF16(_mm512_castsi512_ph(_mm512_shuffle_epi32(_mm512_castph_si512(this->v), imm))); + } + } else if constexpr(VectorBase::template CheckEpi8Shuffle()){ + constexpr std::array::Alignment> shuffleMask = VectorBase::template GetShuffleMaskEpi8(); + if constexpr(std::is_same_v::VectorType, __m128h>) { + __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); + return VectorF16(_mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(this->v), shuffleVec))); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + __m256i shuffleVec = _mm256_loadu_epi8(shuffleMask.data()); + return VectorF16(_mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(this->v)), _mm512_castsi256_si512(shuffleVec))))); + } else { + __m512i shuffleVec = _mm512_loadu_epi8(shuffleMask.data()); + return VectorF16(_mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(this->v), shuffleVec))); + } + } else { + if constexpr(std::is_same_v::VectorType, __m128h>) { + constexpr std::array::Alignment> shuffleMask = VectorBase::template GetShuffleMaskEpi8(); + __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); + return VectorF16(_mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(this->v), shuffleVec))); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetPermuteMaskEpi16(); + __m256i permIdx = _mm256_loadu_epi16(permMask.data()); + return VectorF16(_mm256_castsi256_ph(_mm256_permutexvar_epi16(permIdx, _mm256_castph_si256(this->v)))); + } else { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetPermuteMaskEpi16(); + __m512i permIdx = _mm512_loadu_epi16(permMask.data()); + return VectorF16(_mm512_castsi512_ph(_mm512_permutexvar_epi16(permIdx, _mm512_castph_si512(this->v)))); + } + } + } + + // Public variadic surface — one name per op, arity locked to BatchSize + // (or 2*BatchSize for Dot). Forwards to the *Pack helpers below which + // carry the SIMD bodies and per-(Len,Packing) requires clauses. + template + requires ((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto Normalize(VectorF16 first, Rest... rest) { + return NormalizePack(first, rest...); + } + + template + requires ((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto Length(VectorF16 first, Rest... rest) { + return LengthPack(first, rest...); + } + + template + requires ((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto LengthSq(VectorF16 first, Rest... rest) { + return LengthSqPack(first, rest...); + } + + template + requires ((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == 2 * VectorBase::BatchSize)) + constexpr static auto Dot(VectorF16 first, Rest... rest) { + return DotPack(first, rest...); + } + + private: + constexpr static std::array, VectorBase::BatchSize> NormalizePack( + VectorF16 A, + VectorF16 C, + VectorF16 E, + VectorF16 G + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + VectorF16<1, 8> lenght = LengthNoShuffle(A, E, C, G); + constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1}; + __m128h one = _mm_loadu_ph(oneArr); + VectorF16<8, 1> fLenght(_mm_div_ph(one, lenght.v)); + + VectorF16<8, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,1,1,1,1}}>(); + VectorF16<8, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,3,3,3,3}}>(); + VectorF16<8, 1> fLenghtE = fLenght.template Shuffle<{{4,4,4,4,5,5,5,5}}>(); + VectorF16<8, 1> fLenghtG = fLenght.template Shuffle<{{6,6,6,6,7,7,7,7}}>(); + + return { + _mm_mul_ph(A.v, fLenghtA.v), + _mm_mul_ph(C.v, fLenghtC.v), + _mm_mul_ph(E.v, fLenghtE.v), + _mm_mul_ph(G.v, fLenghtG.v) + }; + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + VectorF16<1, 16> lenght = LengthNoShuffle(A, E, C, G); + constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + __m256h one = _mm256_loadu_ph(oneArr); + VectorF16<16, 1> fLenght(_mm256_div_ph(one, lenght.v)); + + + VectorF16<16, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,1,1,1,1,8,8,8,8,9,9,9,9}}>(); + VectorF16<16, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,3,3,3,3,10,10,10,10,11,11,11,11}}>(); + VectorF16<16, 1> fLenghtE = fLenght.template Shuffle<{{4,4,4,4,5,5,5,5,12,12,12,12,13,13,13,13}}>(); + VectorF16<16, 1> fLenghtG = fLenght.template Shuffle<{{6,6,6,6,7,7,7,7,14,14,14,14,15,15,15,15}}>(); + + return { + _mm256_mul_ph(A.v, fLenghtA.v), + _mm256_mul_ph(C.v, fLenghtC.v), + _mm256_mul_ph(E.v, fLenghtE.v), + _mm256_mul_ph(G.v, fLenghtG.v) + }; + } else { + VectorF16<1, 32> lenght = LengthNoShuffle(A, E, C, G); + constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + __m512h one = _mm512_loadu_ph(oneArr); + VectorF16<32, 1> fLenght(_mm512_div_ph(one, lenght.v)); + VectorF16<32, 1> fLenght2(lenght.v); + + VectorF16<32, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,1,1,1,1,8,8,8,8,9,9,9,9,16,16,16,16,17,17,17,17,24,24,24,24,25,25,25,25}}>(); + VectorF16<32, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,3,3,3,3,10,10,10,10,11,11,11,11,18,18,18,18,19,19,19,19,26,26,26,26,27,27,27,27}}>(); + VectorF16<32, 1> fLenghtE = fLenght.template Shuffle<{{4,4,4,4,5,5,5,5,12,12,12,12,13,13,13,13,20,20,20,20,21,21,21,21,28,28,28,28,29,29,29,29}}>(); + VectorF16<32, 1> fLenghtG = fLenght.template Shuffle<{{6,6,6,6,7,7,7,7,14,14,14,14,15,15,15,15,22,22,22,22,23,23,23,23,30,30,30,30,31,31,31,31}}>(); + + return { + VectorF16(_mm512_mul_ph(A.v, fLenghtA.v)), + VectorF16(_mm512_mul_ph(C.v, fLenghtC.v)), + VectorF16(_mm512_mul_ph(E.v, fLenghtE.v)), + VectorF16(_mm512_mul_ph(G.v, fLenghtG.v)), + }; + } + } + + constexpr static std::array, VectorBase::BatchSize> NormalizePack( + VectorF16 A, + VectorF16 E + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + VectorF16<1, 8> lenght = LengthNoShuffle(A, E); + constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1}; + __m128h one = _mm_loadu_ph(oneArr); + VectorF16<8, 1> fLenght(_mm_div_ph(one, lenght.v)); + + VectorF16<8, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,2,2,3,3}}>(); + VectorF16<8, 1> fLenghtE = fLenght.template Shuffle<{{4,4,5,5,6,6,7,7}}>(); + + return { + _mm_mul_ph(A.v, fLenghtA.v), + _mm_mul_ph(E.v, fLenghtE.v), + }; + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + VectorF16<1, 16> lenght = LengthNoShuffle(A, E); + constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + __m256h one = _mm256_loadu_ph(oneArr); + VectorF16<16, 1> fLenght(_mm256_div_ph(one, lenght.v)); + + VectorF16<16, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11}}>(); + VectorF16<16, 1> fLenghtE = fLenght.template Shuffle<{{4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15}}>(); + + return { + _mm256_mul_ph(A.v, fLenghtA.v), + _mm256_mul_ph(E.v, fLenghtE.v), + }; + } else { + VectorF16<1, 32> lenght = LengthNoShuffle(A, E); + constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + __m512h one = _mm512_loadu_ph(oneArr); + VectorF16<32, 1> fLenght(_mm512_div_ph(one, lenght.v)); + + VectorF16<32, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11,16,16,17,17,18,18,19,19,24,24,25,25,26,26,27,27}}>(); + VectorF16<32, 1> fLenghtE = fLenght.template Shuffle<{{4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15,20,20,21,21,22,22,23,23,28,28,29,29,30,30,31,31}}>(); + + return { + _mm512_mul_ph(A.v, fLenghtA.v), + _mm512_mul_ph(E.v, fLenghtE.v), + }; + } + } + + constexpr static VectorF16<1, Packing*4> LengthPack( + VectorF16 A, + VectorF16 C, + VectorF16 E, + VectorF16 G + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + VectorF16<1, Packing*4> lenghtSq = LengthSqPack(A, C, E, G); + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16<1, Packing*4>(_mm_sqrt_ph(lenghtSq.v)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16<1, Packing*4>(_mm256_sqrt_ph(lenghtSq.v)); + } else { + return VectorF16<1, Packing*4>(_mm512_sqrt_ph(lenghtSq.v)); + } + } + + constexpr static VectorF16<1, Packing*2> LengthPack( + VectorF16 A, + VectorF16 E + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + VectorF16<1, Packing*2> lenghtSq = LengthSqPack(A, E); + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16<1, Packing*2>(_mm_sqrt_ph(lenghtSq.v)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16<1, Packing*2>(_mm256_sqrt_ph(lenghtSq.v)); + } else { + return VectorF16<1, Packing*2>(_mm512_sqrt_ph(lenghtSq.v)); + } + } + + constexpr static VectorF16<1, Packing*4> LengthSqPack( + VectorF16 A, + VectorF16 C, + VectorF16 E, + VectorF16 G + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + return DotPack(A, A, C, C, E, E, G, G); + } + + constexpr static VectorF16<1, Packing*2> LengthSqPack( + VectorF16 A, + VectorF16 E + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + return DotPack(A, A, E, E); + } + + constexpr static VectorF16<1, Packing*4> DotPack( + VectorF16 A0, VectorF16 A1, + VectorF16 C0, VectorF16 C1, + VectorF16 E0, VectorF16 E1, + VectorF16 G0, VectorF16 G1 + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + return DotNoShuffle(A0, A1, E0, E1, C0, C1, G0, G1); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + VectorF16<16, 1> vec(DotNoShuffle(A0, A1, C0, C1, E0, E1, G0, G1).v); + vec = vec.template Shuffle<{{ + 0,1,8,9, + 4,5,12,13, + 2,3,10,11, + 6,7,14,15 + }}>(); + return vec.v; + } else { + VectorF16<32, 1> vec(DotNoShuffle(A0, A1, C0, C1, E0, E1, G0, G1).v); + vec = vec.template Shuffle<{{ + 0,1,8,9, + 16,17,24,25, + + 4,5,12,13, + 20,21,28,29, + + 2,3,10,11, + 18,19,24,25, + + 6,7,14,15, + 22,23,30,31 + }}>(); + return vec.v; + } + } + + constexpr static VectorF16<1, Packing*2> DotPack( + VectorF16 A0, VectorF16 A1, + VectorF16 E0, VectorF16 E1 + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + return DotNoShuffle(A0, A1, E0, E1); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + VectorF16<16, 1> vec(DotNoShuffle(A0, A1, E0, E1).v); + vec = vec.template Shuffle<{{0,1,2,3,8,9,10,11,4,5,6,7,12,13,14,15}}>(); + return vec.v; + } else { + VectorF16<32, 1> vec(DotNoShuffle(A0, A1, E0, E1).v); + vec = vec.template Shuffle<{{0,1,2,3,8,9,10,11,16,17,18,19,24,25,26,27,4,5,6,7,12,13,14,15,20,21,22,23,28,29,30,31}}>(); + return vec.v; + } + } + + + private: + constexpr static VectorF16<1, Packing*4> LengthNoShuffle( + VectorF16 A, + VectorF16 C, + VectorF16 E, + VectorF16 G + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + VectorF16<1, Packing*4> lenghtSq = LengthSqNoShuffle(A, C, E, G); + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16<1, Packing*4>(_mm_sqrt_ph(lenghtSq.v)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16<1, Packing*4>(_mm256_sqrt_ph(lenghtSq.v)); + } else { + return VectorF16<1, Packing*4>(_mm512_sqrt_ph(lenghtSq.v)); + } + } + + constexpr static VectorF16<1, Packing*2> LengthNoShuffle( + VectorF16 A, + VectorF16 E + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + VectorF16<1, Packing*2> lenghtSq = LengthSqNoShuffle(A, E); + if constexpr(std::is_same_v::VectorType, __m128h>) { + return VectorF16<1, Packing*2>(_mm_sqrt_ph(lenghtSq.v)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + return VectorF16<1, Packing*2>(_mm256_sqrt_ph(lenghtSq.v)); + } else { + return VectorF16<1, Packing*2>(_mm512_sqrt_ph(lenghtSq.v)); + } + } + + constexpr static VectorF16<1, Packing*4> LengthSqNoShuffle( + VectorF16 A, + VectorF16 C, + VectorF16 E, + VectorF16 G + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + return DotNoShuffle(A, A, C, C, E, E, G, G); + } + + constexpr static VectorF16<1, Packing*2> LengthSqNoShuffle( + VectorF16 A, + VectorF16 E + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + return DotNoShuffle(A, A, E, E); + } + + + constexpr static VectorF16<1, Packing*4> DotNoShuffle( + VectorF16 A0, VectorF16 A1, + VectorF16 C0, VectorF16 C1, + VectorF16 E0, VectorF16 E1, + VectorF16 G0, VectorF16 G1 + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + __m128h mulA = _mm_mul_ph(A0.v, A1.v); + __m128h mulC = _mm_mul_ph(C0.v, C1.v); + + __m128i row12Temp1 = _mm_unpacklo_epi16(_mm_castph_si128(mulA), _mm_castph_si128(mulC)); // A1 C1 A2 C2 A3 C3 A4 C4 + __m128i row34Temp1 = _mm_unpackhi_epi16(_mm_castph_si128(mulA), _mm_castph_si128(mulC)); // B1 D1 B2 D2 B3 D3 B4 D4 + __m128i row1TempTemp1 = row12Temp1; + + __m128h mulE = _mm_mul_ph(E0.v, E1.v); + __m128h mulG = _mm_mul_ph(G0.v, G1.v); + + __m128i row12Temp2 = _mm_unpacklo_epi16(_mm_castph_si128(mulE), _mm_castph_si128(mulG)); // E1 G1 E2 G2 E3 G3 E4 G4 + __m128i row12Temp2Temp = row12Temp2; + __m128i row34Temp2 = _mm_unpackhi_epi16(_mm_castph_si128(mulE), _mm_castph_si128(mulG)); // F1 H1 F2 H2 F3 H3 F4 H4 + + row12Temp1 = _mm_unpacklo_epi16(row12Temp1, row12Temp2); // A1 E1 C1 G1 A2 E2 C2 G2 + row12Temp2 = _mm_unpacklo_epi16(row34Temp1, row34Temp2); // B1 F1 D1 H1 B2 F2 D2 H2 + row34Temp2 = _mm_unpackhi_epi16(row34Temp1, row34Temp2); // B3 F3 D3 H3 B4 F4 D4 H4 + row34Temp1 = _mm_unpackhi_epi16(row1TempTemp1, row12Temp2Temp); // A3 E3 C3 G3 A4 E4 C4 G4 + + __m128h row1 = _mm_castsi128_ph(_mm_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 E1 F1 C1 D1 G1 H1 + __m128h row2 = _mm_castsi128_ph(_mm_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 E2 F2 C2 D2 G2 H2 + __m128h row3 = _mm_castsi128_ph(_mm_unpacklo_epi16(row34Temp1, row34Temp2));// A3 B3 E3 F3 C3 D3 G3 H3 + __m128h row4 = _mm_castsi128_ph(_mm_unpackhi_epi16(row34Temp1, row34Temp2));// A4 B4 E4 F4 C4 D4 G4 H4 + + row1 = _mm_add_ph(row1, row2); + row1 = _mm_add_ph(row1, row3); + row1 = _mm_add_ph(row1, row4); + + return row1; + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + __m256h mulA = _mm256_mul_ph(A0.v, A1.v); + __m256h mulC = _mm256_mul_ph(C0.v, C1.v); + + __m256i row12Temp1 = _mm256_unpacklo_epi16(_mm256_castph_si256(mulA), _mm256_castph_si256(mulC)); // A1 C1 A2 C2 A3 C3 A4 C4 + __m256i row34Temp1 = _mm256_unpackhi_epi16(_mm256_castph_si256(mulA), _mm256_castph_si256(mulC)); // B1 D1 B2 D2 B3 D3 B4 D4 + __m256i row1TempTemp1 = row12Temp1; + + __m256h mulE = _mm256_mul_ph(E0.v, E1.v); + __m256h mulG = _mm256_mul_ph(G0.v, G1.v); + + __m256i row12Temp2 = _mm256_unpacklo_epi16(_mm256_castph_si256(mulE), _mm256_castph_si256(mulG)); // E1 G1 E2 G2 E3 G3 E4 G4 + __m256i row12Temp2Temp = row12Temp2; + __m256i row34Temp2 = _mm256_unpackhi_epi16(_mm256_castph_si256(mulE), _mm256_castph_si256(mulG)); // F1 H1 F2 H2 F3 H3 F4 H4 + + row12Temp1 = _mm256_unpacklo_epi16(row12Temp1, row12Temp2); // A1 E1 C1 G1 A2 E2 C2 G2 + row12Temp2 = _mm256_unpacklo_epi16(row34Temp1, row34Temp2); // B1 F1 D1 H1 B2 F2 D2 H2 + row34Temp2 = _mm256_unpackhi_epi16(row34Temp1, row34Temp2); // B3 F3 D3 H3 B4 F4 D4 H4 + row34Temp1 = _mm256_unpackhi_epi16(row1TempTemp1, row12Temp2Temp); // A3 E3 C3 G3 A4 E4 C4 G4 + + __m256h row1 = _mm256_castsi256_ph(_mm256_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 E1 F1 C1 D1 G1 H1 + __m256h row2 = _mm256_castsi256_ph(_mm256_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 E2 F2 C2 D2 G2 H2 + __m256h row3 = _mm256_castsi256_ph(_mm256_unpacklo_epi16(row34Temp1, row34Temp2));// A3 B3 E3 F3 C3 D3 G3 H3 + __m256h row4 = _mm256_castsi256_ph(_mm256_unpackhi_epi16(row34Temp1, row34Temp2));// A4 B4 E4 F4 C4 D4 G4 H4 + + row1 = _mm256_add_ph(row1, row2); + row1 = _mm256_add_ph(row1, row3); + row1 = _mm256_add_ph(row1, row4); + + return row1; + } else { + __m512h mulA = _mm512_mul_ph(A0.v, A1.v); + __m512h mulC = _mm512_mul_ph(C0.v, C1.v); + + __m512i row12Temp1 = _mm512_unpacklo_epi16(_mm512_castph_si512(mulA), _mm512_castph_si512(mulC)); // A1 C1 A2 C2 A3 C3 A4 C4 + __m512i row34Temp1 = _mm512_unpackhi_epi16(_mm512_castph_si512(mulA), _mm512_castph_si512(mulC)); // B1 D1 B2 D2 B3 D3 B4 D4 + __m512i row1TempTemp1 = row12Temp1; + + __m512h mulE = _mm512_mul_ph(E0.v, E1.v); + __m512h mulG = _mm512_mul_ph(G0.v, G1.v); + + __m512i row12Temp2 = _mm512_unpacklo_epi16(_mm512_castph_si512(mulE), _mm512_castph_si512(mulG)); // E1 G1 E2 G2 E3 G3 E4 G4 + __m512i row12Temp2Temp = row12Temp2; + __m512i row34Temp2 = _mm512_unpackhi_epi16(_mm512_castph_si512(mulE), _mm512_castph_si512(mulG)); // F1 H1 F2 H2 F3 H3 F4 H4 + + row12Temp1 = _mm512_unpacklo_epi16(row12Temp1, row12Temp2); // A1 E1 C1 G1 A2 E2 C2 G2 + row12Temp2 = _mm512_unpacklo_epi16(row34Temp1, row34Temp2); // B1 F1 D1 H1 B2 F2 D2 H2 + row34Temp2 = _mm512_unpackhi_epi16(row34Temp1, row34Temp2); // B3 F3 D3 H3 B4 F4 D4 H4 + row34Temp1 = _mm512_unpackhi_epi16(row1TempTemp1, row12Temp2Temp); // A3 E3 C3 G3 A4 E4 C4 G4 + + __m512h row1 = _mm512_castsi512_ph(_mm512_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 E1 F1 C1 D1 G1 H1 + __m512h row2 = _mm512_castsi512_ph(_mm512_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 E2 F2 C2 D2 G2 H2 + __m512h row3 = _mm512_castsi512_ph(_mm512_unpacklo_epi16(row34Temp1, row34Temp2));// A3 B3 E3 F3 C3 D3 G3 H3 + __m512h row4 = _mm512_castsi512_ph(_mm512_unpackhi_epi16(row34Temp1, row34Temp2));// A4 B4 E4 F4 C4 D4 G4 H4 + + row1 = _mm512_add_ph(row1, row2); + row1 = _mm512_add_ph(row1, row3); + row1 = _mm512_add_ph(row1, row4); + + return row1; + } + } + + constexpr static VectorF16<1, Packing*2> DotNoShuffle( + VectorF16 A0, VectorF16 A1, + VectorF16 E0, VectorF16 E1 + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + if constexpr(std::is_same_v::VectorType, __m128h>) { + __m128h mulA = _mm_mul_ph(A0.v, A1.v); + __m128h mulE = _mm_mul_ph(E0.v, E1.v); + __m128i row12Temp1 = _mm_unpacklo_epi16(_mm_castph_si128(mulA), _mm_castph_si128(mulE)); // A1 E1 A2 E2 B1 F1 B2 F2 + __m128i row12Temp2 = _mm_unpackhi_epi16(_mm_castph_si128(mulA), _mm_castph_si128(mulE)); // C1 G1 C2 G2 D1 H1 D2 H2 + __m128i row12Temp1Temp = row12Temp1; + + row12Temp1 = _mm_unpacklo_epi16(row12Temp1, row12Temp2); // A1 C1 E1 G1 A2 C2 E2 G2 + row12Temp2 = _mm_unpackhi_epi16(row12Temp1Temp, row12Temp2); // B1 D1 F1 H1 B2 D2 F2 H2 + + __m128h row1 = _mm_castsi128_ph(_mm_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 C1 D1 E1 F1 G1 H1 + __m128h row2 = _mm_castsi128_ph(_mm_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 C2 D2 E2 F2 G2 H2 + + return _mm_add_ph(row1, row2); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + __m256h mulA = _mm256_mul_ph(A0.v, A1.v); + __m256h mulE = _mm256_mul_ph(E0.v, E1.v); + + __m256i row12Temp1 = _mm256_unpacklo_epi16(_mm256_castph_si256(mulA), _mm256_castph_si256(mulE)); // A1 E1 A2 E2 B1 F1 B2 F2 + + __m256i row12Temp2 = _mm256_unpackhi_epi16(_mm256_castph_si256(mulA), _mm256_castph_si256(mulE)); // C1 G1 C2 G2 D1 H1 D2 H2 + __m256i row12Temp1Temp = row12Temp1; + + row12Temp1 = _mm256_unpacklo_epi16(row12Temp1, row12Temp2); // A1 C1 E1 G1 A2 C2 E2 G2 + row12Temp2 = _mm256_unpackhi_epi16(row12Temp1Temp, row12Temp2); // B1 D1 F1 H1 B2 D2 F2 H2 + + __m256h row1 = _mm256_castsi256_ph(_mm256_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 C1 D1 E1 F1 G1 H1 + __m256h row2 = _mm256_castsi256_ph(_mm256_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 C2 D2 E2 F2 G2 H2 + __m256h result = _mm256_add_ph(row1, row2); + + return result; + } else { + __m512h mulA = _mm512_mul_ph(A0.v, A1.v); + __m512h mulE = _mm512_mul_ph(E0.v, E1.v); + __m512i row12Temp1 = _mm512_unpacklo_epi16(_mm512_castph_si512(mulA), _mm512_castph_si512(mulE)); // A1 E1 A2 E2 B1 F1 B2 F2 + __m512i row12Temp2 = _mm512_unpackhi_epi16(_mm512_castph_si512(mulA), _mm512_castph_si512(mulE)); // C1 G1 C2 G2 D1 H1 D2 H2 + __m512i row12Temp1Temp = row12Temp1; + + row12Temp1 = _mm512_unpacklo_epi16(row12Temp1, row12Temp2); // A1 C1 E1 G1 A2 C2 E2 G2 + row12Temp2 = _mm512_unpackhi_epi16(row12Temp1Temp, row12Temp2); // B1 D1 F1 H1 B2 D2 F2 H2 + + __m512h row1 = _mm512_castsi512_ph(_mm512_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 C1 D1 E1 F1 G1 H1 + __m512h row2 = _mm512_castsi512_ph(_mm512_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 C2 D2 E2 F2 G2 H2 + __m512h result = _mm512_add_ph(row1, row2); + + return result; + } + } + public: + + template ShuffleValues> + constexpr static VectorF16 Blend(VectorF16 a, VectorF16 b) { + constexpr auto mask = VectorBase::template GetBlendMaskEpi16(); + if constexpr(std::is_same_v::VectorType, __m128h>) { + return _mm_castsi128_ph(_mm_blend_epi16(_mm_castph_si128(a.v), _mm_castph_si128(b.v), mask)); + } else if constexpr(std::is_same_v::VectorType, __m256h>) { + #ifndef __AVX512BW__ + #ifndef __AVX512VL__ + static_assert(false, "No __AVX512BW__ and __AVX512VL__ support"); + #endif + #endif + return _mm256_castsi256_ph(_mm256_mask_blend_epi16(mask, _mm256_castph_si256(a.v), _mm256_castph_si256(b.v))); + } else { + return _mm512_castsi512_ph(_mm512_mask_blend_epi16(mask, _mm512_castph_si512(a.v), _mm512_castph_si512(b.v))); + } + } + + constexpr static VectorF16 Rotate(VectorF16<3, Packing> v, VectorF16<4, Packing> q) requires(Len == 3) { + VectorF16<3, Packing> qv(q); + VectorF16 t = Cross(qv, v) * _Float16(2); + return v + t * q.template Shuffle<{{3,3,3,3}}>() + Cross(qv, t); + } + + constexpr static VectorF16<4, 2> RotatePivot(VectorF16<3, Packing> v, VectorF16<4, Packing> q, VectorF16<3, Packing> pivot) requires(Len == 3) { + VectorF16 translated = v - pivot; + VectorF16<3, Packing> qv(q.v); + VectorF16 t = Cross(qv, translated) * _Float16(2); + VectorF16 rotated = translated + t * q.template Shuffle<{{3,3,3,3}}>() + Cross(qv, t); + return rotated + pivot; + } + + constexpr static VectorF16<4, Packing> QuanternionFromEuler(VectorF16<3, Packing> EulerHalf) requires(Len == 4) { + std::tuple, VectorF16<3, Packing>> sinCos = EulerHalf.SinCos(); + VectorF16<4, Packing> sin = std::get<0>(sinCos); + VectorF16<4, Packing> cos = std::get<1>(sinCos); + + VectorF16<4, Packing> row1 = cos.template Shuffle<{{0,0,0,0}}>(); + row1 = Blend<{{0,1,1,1}}>(sin, row1); + + VectorF16<4, Packing> row2 = cos.template Shuffle<{{1,1,1,1}}>(); + row2 = Blend<{{1,0,1,1}}>(sin, row2); + + row1 *= row2; + + VectorF16<4, Packing> row3 = cos.template Shuffle<{{2,2,2,2}}>(); + row3 = Blend<{{1,1,0,1}}>(sin, row3); + + row1 *= row3; + + VectorF16<4, Packing> row4 = sin.template Shuffle<{{0,0,0,0}}>(); + row4 = Blend<{{0,1,1,1}}>(cos, row4); + + VectorF16<4, Packing> row5 = sin.template Shuffle<{{1,1,1,1}}>(); + row5 = Blend<{{1,0,1,1}}>(cos, row5); + + row4 *= row5; + + VectorF16<4, Packing> row6 = sin.template Shuffle<{{2,2,2,2}}>(); + row6 = Blend<{{1,1,0,1}}>(cos, row6); + row6 = row6.template Negate<{{true,false,true,false}}>(); + + row1 = MulitplyAdd(row4, row6, row1); + + return row1; + } + }; +} + + +#elif !defined(__x86_64) && defined(__FLT16_MAX__) +// Scalar software fallback for non-x86_64 targets that still have _Float16. +namespace Crafter { + export template + struct VectorF16 : public VectorBase { + template + friend struct VectorF16; + using Base = VectorBase; + static constexpr std::uint8_t NElems = Base::AlignmentElement; + + constexpr VectorF16() = default; + constexpr VectorF16(typename Base::VectorType vv) { + this->v = vv; + } + constexpr VectorF16(const _Float16* vB) { Load(vB); } + constexpr VectorF16(_Float16 val) { + for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = val; + } + + constexpr void Load(const _Float16* vB) { + for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = vB[i]; + } + constexpr void Store(_Float16* vB) const { + for (std::uint8_t i = 0; i < NElems; ++i) vB[i] = this->v[i]; + } + + template + constexpr std::array<_Float16, NElems> Store() const { + std::array<_Float16, NElems> r{}; + Store(r.data()); + return r; + } + + template + constexpr operator VectorF16() const { + VectorF16 r; + const std::uint8_t copyLen = (BLen < Len) ? BLen : Len; + const std::uint8_t copyPack = (BPacking < Packing) ? BPacking : Packing; + for (std::uint8_t p = 0; p < copyPack; ++p) + for (std::uint8_t i = 0; i < copyLen; ++i) + r.v[p * BLen + i] = this->v[p * Len + i]; + return r; + } + + constexpr VectorF16 operator+(VectorF16 b) const { + VectorF16 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] + b.v[i]; + return r; + } + constexpr VectorF16 operator-(VectorF16 b) const { + VectorF16 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] - b.v[i]; + return r; + } + constexpr VectorF16 operator*(VectorF16 b) const { + VectorF16 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] * b.v[i]; + return r; + } + constexpr VectorF16 operator/(VectorF16 b) const { + VectorF16 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] / b.v[i]; + return r; + } + + constexpr void operator+=(VectorF16 b) { for (std::uint8_t i=0;iv[i] += b.v[i]; } + constexpr void operator-=(VectorF16 b) { for (std::uint8_t i=0;iv[i] -= b.v[i]; } + constexpr void operator*=(VectorF16 b) { for (std::uint8_t i=0;iv[i] *= b.v[i]; } + constexpr void operator/=(VectorF16 b) { for (std::uint8_t i=0;iv[i] /= b.v[i]; } + + constexpr VectorF16 operator+(_Float16 b) const { return *this + VectorF16(b); } + constexpr VectorF16 operator-(_Float16 b) const { return *this - VectorF16(b); } + constexpr VectorF16 operator*(_Float16 b) const { return *this * VectorF16(b); } + constexpr VectorF16 operator/(_Float16 b) const { return *this / VectorF16(b); } + constexpr void operator+=(_Float16 b) { *this += VectorF16(b); } + constexpr void operator-=(_Float16 b) { *this -= VectorF16(b); } + constexpr void operator*=(_Float16 b) { *this *= VectorF16(b); } + constexpr void operator/=(_Float16 b) { *this /= VectorF16(b); } + + constexpr VectorF16 operator-() const { + VectorF16 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = -this->v[i]; + return r; + } + + constexpr bool operator==(VectorF16 b) const { + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + if (this->v[p * Len + i] != b.v[p * Len + i]) return false; + return true; + } + constexpr bool operator!=(VectorF16 b) const { return !(*this == b); } + + template + constexpr VectorF16 ExtractLo() const { + VectorF16 r; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < ExtractLen; ++i) + r.v[p * ExtractLen + i] = this->v[p * Len + i]; + return r; + } + + // Transcendentals are computed via float since libstdc++ doesn't always + // provide overloads for _Float16; the result is rounded back to half. + constexpr VectorF16 Cos() const { + VectorF16 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = static_cast<_Float16>(std::cos(static_cast(this->v[i]))); + return r; + } + constexpr VectorF16 Sin() const { + VectorF16 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = static_cast<_Float16>(std::sin(static_cast(this->v[i]))); + return r; + } + constexpr std::tuple, VectorF16> SinCos() const { + return { Sin(), Cos() }; + } + + template values> + constexpr VectorF16 Negate() const { + VectorF16 r; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + r.v[p * Len + i] = values[i] ? -this->v[p * Len + i] : this->v[p * Len + i]; + return r; + } + + static constexpr VectorF16 MulitplyAdd(VectorF16 a, VectorF16 b, VectorF16 add) { + VectorF16 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = a.v[i] * b.v[i] + add.v[i]; + return r; + } + static constexpr VectorF16 MulitplySub(VectorF16 a, VectorF16 b, VectorF16 sub) { + VectorF16 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = a.v[i] * b.v[i] - sub.v[i]; + return r; + } + + constexpr static VectorF16 Cross(VectorF16 a, VectorF16 b) requires(Len == 3) { + VectorF16 r; + for (std::uint8_t p = 0; p < Packing; ++p) { + const std::uint8_t base = p * 3; + r.v[base + 0] = a.v[base + 1] * b.v[base + 2] - a.v[base + 2] * b.v[base + 1]; + r.v[base + 1] = a.v[base + 2] * b.v[base + 0] - a.v[base + 0] * b.v[base + 2]; + r.v[base + 2] = a.v[base + 0] * b.v[base + 1] - a.v[base + 1] * b.v[base + 0]; + } + return r; + } + + template ShuffleValues> + constexpr VectorF16 Shuffle() const { + VectorF16 r; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + r.v[p * Len + i] = this->v[p * Len + ShuffleValues[i]]; + return r; + } + + template ShuffleValues> + constexpr static VectorF16 Blend(VectorF16 a, VectorF16 b) { + VectorF16 r; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + r.v[p * Len + i] = ShuffleValues[i] ? b.v[p * Len + i] : a.v[p * Len + i]; + return r; + } + + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto LengthSq(VectorF16 first, Rest... rest) { + constexpr std::uint8_t N = VectorBase::BatchSize; + VectorF16<1, static_cast(Packing * N)> r; + std::array, N> args{ first, rest... }; + for (std::uint8_t i = 0; i < N; ++i) + for (std::uint8_t p = 0; p < Packing; ++p) { + _Float16 acc = _Float16(0); + for (std::uint8_t k = 0; k < Len; ++k) { + _Float16 x = args[i].v[p * Len + k]; + acc += x * x; + } + r.v[i * Packing + p] = acc; + } + return r; + } + + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto Length(VectorF16 first, Rest... rest) { + auto sq = LengthSq(first, rest...); + for (std::uint8_t i = 0; i < decltype(sq)::NElems; ++i) + sq.v[i] = static_cast<_Float16>(std::sqrt(static_cast(sq.v[i]))); + return sq; + } + + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto Normalize(VectorF16 first, Rest... rest) { + auto normOne = [](VectorF16 u) { + VectorF16 out; + for (std::uint8_t p = 0; p < Packing; ++p) { + float acc = 0.0f; + for (std::uint8_t k = 0; k < Len; ++k) { + float x = static_cast(u.v[p * Len + k]); + acc += x * x; + } + _Float16 invLen = acc > 0.0f ? static_cast<_Float16>(1.0f / std::sqrt(acc)) : _Float16(0); + for (std::uint8_t k = 0; k < Len; ++k) + out.v[p * Len + k] = u.v[p * Len + k] * invLen; + } + return out; + }; + return std::array, VectorBase::BatchSize>{ normOne(first), normOne(rest)... }; + } + + constexpr static VectorF16 Rotate(VectorF16<3, Packing> v, VectorF16<4, Packing> q) requires(Len == 3) { + VectorF16<3, Packing> qv; + VectorF16<3, Packing> qwBroadcast; + for (std::uint8_t p = 0; p < Packing; ++p) { + qv.v[p * 3 + 0] = q.v[p * 4 + 0]; + qv.v[p * 3 + 1] = q.v[p * 4 + 1]; + qv.v[p * 3 + 2] = q.v[p * 4 + 2]; + for (std::uint8_t i = 0; i < 3; ++i) qwBroadcast.v[p * 3 + i] = q.v[p * 4 + 3]; + } + VectorF16<3, Packing> t = Cross(qv, v) * _Float16(2); + return v + t * qwBroadcast + Cross(qv, t); + } + + constexpr static VectorF16<3, Packing> RotatePivot(VectorF16<3, Packing> v, VectorF16<4, Packing> q, VectorF16<3, Packing> pivot) requires(Len == 3) { + VectorF16<3, Packing> translated = v - pivot; + return Rotate(translated, q) + pivot; + } + + constexpr static VectorF16<4, Packing> QuanternionFromEuler(VectorF16<3, Packing> eulerHalf) requires(Len == 4) { + VectorF16<4, Packing> r; + for (std::uint8_t p = 0; p < Packing; ++p) { + float roll = static_cast(eulerHalf.v[p * 3 + 0]); + float pitch = static_cast(eulerHalf.v[p * 3 + 1]); + float yaw = static_cast(eulerHalf.v[p * 3 + 2]); + float sr = std::sin(roll), cr = std::cos(roll); + float sp = std::sin(pitch), cp = std::cos(pitch); + float sy = std::sin(yaw), cy = std::cos(yaw); + r.v[p * 4 + 0] = static_cast<_Float16>(sr * cp * cy - cr * sp * sy); + r.v[p * 4 + 1] = static_cast<_Float16>(cr * sp * cy + sr * cp * sy); + r.v[p * 4 + 2] = static_cast<_Float16>(cr * cp * sy - sr * sp * cy); + r.v[p * 4 + 3] = static_cast<_Float16>(cr * cp * cy + sr * sp * sy); + } + return r; + } + }; +} +#endif + +#if defined(__FLT16_MAX__) && (!defined(__x86_64) || defined(__AVX512FP16__)) +export template +struct std::formatter> : std::formatter { + constexpr auto format(const Crafter::VectorF16& obj, format_context& ctx) const { + std::array<_Float16, Crafter::VectorF16::AlignmentElement> vec = obj.template Store(); + std::string out = "{"; + for(std::uint32_t i = 0; i < Packing; i++) { + out += "{"; + for(std::uint32_t i2 = 0; i2 < Len; i2++) { + out += std::format("{}", static_cast(vec[i * Len + i2])); + if (i2 + 1 < Len) out += ","; + } + out += "}"; + } + out += "}"; + return std::formatter::format(out, ctx); + } +}; #endif \ No newline at end of file diff --git a/interfaces/Crafter.Math-VectorF32.cppm b/interfaces/Crafter.Math-VectorF32.cppm index ecb91a1..f54420f 100755 --- a/interfaces/Crafter.Math-VectorF32.cppm +++ b/interfaces/Crafter.Math-VectorF32.cppm @@ -1,2214 +1,2214 @@ -//SPDX-License-Identifier: LGPL-3.0-only -//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® - -module; -#ifdef __x86_64 -#include -#endif -#ifdef __wasm_simd128__ -#include -#endif -#ifdef __riscv_vector -#include -#endif -export module Crafter.Math:VectorF32; -import std; -import :Common; - -namespace Crafter { -#ifdef __x86_64 - export template - struct VectorF32 : public VectorBase { - template - friend struct VectorF32; - - constexpr VectorF32() = default; - constexpr VectorF32(VectorBase::VectorType v) { - this->v = v; - } - constexpr VectorF32(const float* vB) { - Load(vB); - }; - constexpr VectorF32(const _Float16* vB) { - Load(vB); - }; - constexpr VectorF32(float val) { - if constexpr(std::is_same_v::VectorType, __m128>) { - this->v = _mm_set1_ps(val); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - this->v = _mm256_set1_ps(val); - } else { - this->v = _mm512_set1_ps(val); - } - }; - constexpr void Load(const float* vB) { - if constexpr(std::is_same_v::VectorType, __m128>) { - this->v = _mm_loadu_ps(vB); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - this->v = _mm256_loadu_ps(vB); - } else { - this->v = _mm512_loadu_ps(vB); - } - } - constexpr void Store(float* vB) const { - if constexpr(std::is_same_v::VectorType, __m128>) { - _mm_storeu_ps(vB, this->v); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - _mm256_storeu_ps(vB, this->v); - } else { - _mm512_storeu_ps(vB, this->v); - } - } - - constexpr void Load(const _Float16* vB) { - #ifdef __F16C__ - if constexpr (std::is_same_v::VectorType, __m128>) { - this->v = _mm_cvtph_ps(_mm_loadl_epi64(reinterpret_cast(vB))); - } else if constexpr (std::is_same_v::VectorType, __m256>) { - this->v = _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(vB))); - } else { - this->v = _mm512_cvtph_ps(_mm256_loadu_si256(reinterpret_cast(vB))); - } - #else - alignas(64) float tmp[Len]; - for (int i = 0; i < Len; ++i) - tmp[i] = static_cast(vB[i]); - if constexpr (std::is_same_v::VectorType, __m128>) { - this->v = _mm_load_ps(tmp); - } else if constexpr (std::is_same_v::VectorType, __m256>) { - this->v = _mm256_load_ps(tmp); - } else { - this->v = _mm512_load_ps(tmp); - } - #endif - } - - constexpr void Store(_Float16* vB) const { - #ifdef __F16C__ - if constexpr (std::is_same_v::VectorType, __m128>) { - _mm_storel_epi64(reinterpret_cast<__m128i*>(vB), _mm_cvtps_ph(this->v, _MM_FROUND_TO_NEAREST_INT)); - } else if constexpr (std::is_same_v::VectorType, __m256>) { - _mm_storeu_si128(reinterpret_cast<__m128i*>(vB), _mm256_cvtps_ph(this->v, _MM_FROUND_TO_NEAREST_INT)); - } else { - _mm256_storeu_si256(reinterpret_cast<__m256i*>(vB), _mm512_cvtps_ph(this->v, _MM_FROUND_TO_NEAREST_INT)); - } - #else - alignas(64) float tmp[Len]; - if constexpr (std::is_same_v::VectorType, __m128>) { - _mm_store_ps(tmp, this->v); - } else if constexpr (std::is_same_v::VectorType, __m256>) { - _mm256_store_ps(tmp, this->v); - } else { - _mm512_store_ps(tmp, this->v); - } - for (int i = 0; i < Len; ++i) - vB[i] = static_cast<_Float16>(tmp[i]); - #endif - } - - template - constexpr std::array::AlignmentElement> Store() const { - std::array::AlignmentElement> returnArray; - Store(returnArray.data()); - return returnArray; - } - - template - constexpr operator VectorF32() const { - if constexpr (Len == BLen) { - if constexpr(std::is_same_v::VectorType, __m256> && std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm256_castps256_ps128(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m512> && std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm512_castps512_ps128(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m512> && std::is_same_v::VectorType, __m256>) { - return VectorF32(_mm512_castps512_ps256(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m128> && std::is_same_v::VectorType, __m256>) { - return VectorF32(_mm256_castps128_ps256(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m128> && std::is_same_v::VectorType, __m512>) { - return VectorF32(_mm512_castps128_ps512(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m256> && std::is_same_v::VectorType, __m512>) { - return VectorF32(_mm512_castps256_ps512(this->v)); - } else { - return VectorF32(this->v); - } - } else if constexpr (BLen <= Len) { - return this->template ExtractLo(); - } else { - if constexpr(std::is_same_v::VectorType, __m128>) { - if constexpr(std::is_same_v::VectorType, __m128>) { - constexpr std::array::Alignment> shuffleMask = VectorBase::template GetExtractLoMaskEpi8(); - __m128i shuffleVec = _mm_loadu_si128(reinterpret_cast(shuffleMask.data())); - return VectorF32(_mm_castsi128_ps(_mm_shuffle_epi8(_mm_castps_si128(this->v), shuffleVec))); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - constexpr std::array::AlignmentElement> permMask =VectorBase::template GetExtractLoMaskepi32(); - __m256i permIdx = _mm256_loadu_si256(reinterpret_cast(permMask.data())); - __m256i result = _mm256_permutexvar_epi32(permIdx, _mm_castps_si256(this->v)); - return VectorF32(_mm_castsi128_ps(_mm256_castsi256_si128(result))); - #ifdef __AVX512F__ - } else { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); - __m512i permIdx = _mm512_loadu_epi32(permMask.data()); - __m512i result = _mm512_permutexvar_epi32(permIdx, _mm512_castps_si512(this->v)); - return VectorF32(_mm_castsi128_ps(_mm512_castsi512_si128(result))); - #endif - } - } else if constexpr(std::is_same_v::VectorType, __m256>) { - if constexpr(std::is_same_v::VectorType, __m128>) { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); - __m256i permIdx = _mm256_loadu_si256(reinterpret_cast(permMask.data())); - __m256i result = _mm256_permutexvar_epi32(permIdx, _mm256_castsi128_si256(_mm_castps_si128(this->v))); - return VectorF32(_mm256_castsi256_ps(result)); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); - __m256i permIdx = _mm256_loadu_si256(reinterpret_cast(permMask.data())); - __m256i result = _mm256_permutexvar_epi32(permIdx, _mm256_castps_si256(this->v)); - return VectorF32(_mm256_castsi256_ps(result)); - #ifdef __AVX512F__ - } else { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); - __m256i permIdx = _mm512_loadu_epi32(permMask.data()); - __m256i result = _mm512_permutexvar_epi32(permIdx, _mm512_castsi512_si256(_mm512_castps_si512(this->v))); - return VectorF32(_mm256_castsi256_ps(result)); - #endif - } - #ifdef __AVX512F__ - } else { - if constexpr(std::is_same_v::VectorType, __m128>) { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); - __m512i permIdx = _mm512_loadu_epi32(permMask.data()); - __m512i result = _mm512_permutexvar_epi32(permIdx, _mm512_castsi128_si512(_mm_castps_si128(this->v))); - return VectorF32(_mm512_castsi512_ps(result)); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); - __m512i permIdx = _mm512_loadu_epi32(permMask.data()); - __m512i result = _mm512_permutexvar_epi32(permIdx, _mm512_castsi256_si512(_mm256_castps_si256(this->v))); - return VectorF32(_mm512_castsi512_ps(result)); - } else { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); - __m512i permIdx = _mm512_loadu_epi32(permMask.data()); - __m512i result = _mm512_permutexvar_epi32(permIdx, _mm512_castps_si512(this->v)); - return VectorF32(_mm512_castsi512_ps(result)); - } - #endif - } - } - } - - constexpr VectorF32 operator+(VectorF32 b) const { - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm_add_ps(this->v, b.v)); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32(_mm256_add_ps(this->v, b.v)); - } else { - return VectorF32(_mm512_add_ps(this->v, b.v)); - } - } - - constexpr VectorF32 operator-(VectorF32 b) const { - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm_sub_ps(this->v, b.v)); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32(_mm256_sub_ps(this->v, b.v)); - } else { - return VectorF32(_mm512_sub_ps(this->v, b.v)); - } - } - - constexpr VectorF32 operator*(VectorF32 b) const { - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm_mul_ps(this->v, b.v)); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32(_mm256_mul_ps(this->v, b.v)); - } else { - return VectorF32(_mm512_mul_ps(this->v, b.v)); - } - } - - constexpr VectorF32 operator/(VectorF32 b) const { - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm_div_ps(this->v, b.v)); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32(_mm256_div_ps(this->v, b.v)); - } else { - return VectorF32(_mm512_div_ps(this->v, b.v)); - } - } - - - constexpr void operator+=(VectorF32 b) { - if constexpr(std::is_same_v::VectorType, __m128>) { - this->v = _mm_add_ps(this->v, b.v); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - this->v = _mm256_add_ps(this->v, b.v); - } else { - this->v = _mm512_add_ps(this->v, b.v); - } - } - - constexpr void operator-=(VectorF32 b) { - if constexpr(std::is_same_v::VectorType, __m128>) { - this->v = _mm_sub_ps(this->v, b.v); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - this->v = _mm256_sub_ps(this->v, b.v); - } else { - this->v = _mm512_sub_ps(this->v, b.v); - } - } - - constexpr void operator*=(VectorF32 b) { - if constexpr(std::is_same_v::VectorType, __m128>) { - this->v = _mm_mul_ps(this->v, b.v); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - this->v = _mm256_mul_ps(this->v, b.v); - } else { - this->v = _mm512_mul_ps(this->v, b.v); - } - } - - constexpr void operator/=(VectorF32 b) { - if constexpr(std::is_same_v::VectorType, __m128>) { - this->v = _mm_div_ps(this->v, b.v); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - this->v = _mm256_div_ps(this->v, b.v); - } else { - this->v = _mm512_div_ps(this->v, b.v); - } - } - - constexpr VectorF32 operator+(float b) { - VectorF32 vB(b); - return *this + vB; - } - - constexpr VectorF32 operator-(float b) { - VectorF32 vB(b); - return *this - vB; - } - - constexpr VectorF32 operator*(float b) { - VectorF32 vB(b); - return *this * vB; - } - - constexpr VectorF32 operator/(float b) { - VectorF32 vB(b); - return *this / vB; - } - - constexpr void operator+=(float b) { - VectorF32 vB(b); - *this += vB; - } - - constexpr void operator-=(float b) { - VectorF32 vB(b); - *this -= vB; - } - - constexpr void operator*=(float b) { - VectorF32 vB(b); - *this *= vB; - } - - constexpr void operator/=(float b) { - VectorF32 vB(b); - *this /= vB; - } - - constexpr VectorF32 operator-(){ - return Negate::GetAllTrue()>(); - } - - constexpr bool operator==(VectorF32 b) const { - if constexpr(std::is_same_v::VectorType, __m128>) { - #ifdef __AVX512VL__ - return _mm_cmp_ps_mask(this->v, b.v, _CMP_EQ_OQ) == 0xF; - #else - return _mm_movemask_ps(_mm_cmpeq_ps(this->v, b.v)) == 0xF; - #endif - } else if constexpr(std::is_same_v::VectorType, __m256>) { - #ifdef __AVX512VL__ - return _mm256_cmp_ps_mask(this->v, b.v, _CMP_EQ_OQ) == 0xFF; - #else - return _mm256_movemask_ps(_mm256_cmp_ps(this->v, b.v, _CMP_EQ_OQ)) == 0xFF; - #endif - } else { - return _mm512_cmp_ps_mask(this->v, b.v, _CMP_EQ_OQ) == 0xFFFF; - } - } - - constexpr bool operator!=(VectorF32 b) const { - return !(*this == b); - } - - template - constexpr VectorF32 ExtractLo() const { - if constexpr(Packing > 1) { - if constexpr(std::is_same_v::VectorType, __m128>) { - constexpr std::array::Alignment> shuffleMask = VectorBase::template GetExtractLoMaskEpi8(); - __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); - return VectorF32(_mm_castsi128_ps(_mm_shuffle_epi8(_mm_castps_si128(this->v), shuffleVec))); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); - __m256i permIdx = _mm256_loadu_si256(reinterpret_cast(permMask.data())); - __m256i result = _mm256_permutexvar_epi32(permIdx, _mm256_castps_si256(this->v)); - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm256_castps256_ps128(_mm256_castsi256_ps(result))); - } else { - return VectorF32(_mm256_castsi256_ps(result)); - } - } else { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); - __m512i permIdx = _mm512_loadu_epi32(permMask.data()); - __m512i result = _mm512_permutexvar_epi32(permIdx, _mm512_castps_si512(this->v)); - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm512_castps512_ps128(_mm512_castsi512_ps(result))); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32(_mm512_castps512_ps256(_mm512_castsi512_ps(result))); - } else { - return VectorF32(_mm512_castsi512_ps(result)); - } - } - } else { - if constexpr(std::is_same_v::VectorType, __m256> && std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm256_castps256_ps128(this->v)); - #ifdef __AVX512F__ - } else if constexpr(std::is_same_v::VectorType, __m512> && std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm512_castps512_ps128(this->v)); - } else if constexpr(std::is_same_v::VectorType, __m512> && std::is_same_v::VectorType, __m256>) { - return VectorF32(_mm512_castps512_ps256(this->v)); - #endif - } else { - return VectorF32(this->v); - } - } - } - - constexpr VectorF32 Cos() { - if constexpr (std::is_same_v::VectorType, __m128>) { - return VectorF32(VectorBase::cos_f32x4(this->v)); - } else if constexpr (std::is_same_v::VectorType, __m256>) { - return VectorF32(VectorBase::cos_f32x8(this->v)); - #ifdef __AVX512F__ - } else { - return VectorF32(VectorBase::cos_f32x16(this->v)); - #endif - } - } - - constexpr VectorF32 Sin() { - if constexpr (std::is_same_v::VectorType, __m128>) { - return VectorF32(VectorBase::sin_f32x4(this->v)); - } else if constexpr (std::is_same_v::VectorType, __m256>) { - return VectorF32(VectorBase::sin_f32x8(this->v)); - #ifdef __AVX512F__ - } else { - return VectorF32(VectorBase::sin_f32x16(this->v)); - #endif - } - } - - std::tuple, VectorF32> SinCos() { - if constexpr (std::is_same_v::VectorType, __m128>) { - __m128 s, c; - VectorBase::sincos_f32x4(this->v, s, c); - return { - VectorF32(s), - VectorF32(c) - }; - - } else if constexpr (std::is_same_v::VectorType, __m256>) { - __m256 s, c; - VectorBase::sincos_f32x8(this->v, s, c); - return { - VectorF32(s), - VectorF32(c) - }; - #ifdef __AVX512F__ - } else { - __m512 s, c; - VectorBase::sincos_f32x16(this->v, s, c); - return { - VectorF32(s), - VectorF32(c) - }; - #endif - } - } - - template values> - constexpr VectorF32 Negate() const { - std::array::AlignmentElement> mask = VectorBase::template GetNegateMask(); - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm_castsi128_ps(_mm_xor_si128(_mm_castps_si128(this->v), _mm_loadu_si128(reinterpret_cast<__m128i*>(mask.data()))))); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32(_mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(this->v), _mm256_loadu_si256(reinterpret_cast<__m256i*>(mask.data()))))); - #ifdef __AVX512F__ - } else { - return VectorF32(_mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(this->v), _mm512_loadu_epi32(mask.data())))); - #endif - } - } - - static constexpr VectorF32 MulitplyAdd(VectorF32 a, VectorF32 b, VectorF32 add) { - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm_fmadd_ps(a.v, b.v, add.v)); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32(_mm256_fmadd_ps(a.v, b.v, add.v)); - #ifdef __AVX512F__ - } else { - return VectorF32(_mm512_fmadd_ps(a.v, b.v, add.v)); - #endif - } - } - - static constexpr VectorF32 MulitplySub(VectorF32 a, VectorF32 b, VectorF32 sub) { - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm_fmsub_ps(a.v, b.v, sub.v)); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32(_mm256_fmsub_ps(a.v, b.v, sub.v)); - #ifdef __AVX512F__ - } else { - return VectorF32(_mm512_fmsub_ps(a.v, b.v, sub.v)); - #endif - } - } - - constexpr static VectorF32 Cross(VectorF32 a, VectorF32 b) requires(Len == 3) { - VectorF32 row1 = a.template Shuffle<{{1,2,0}}>(); - VectorF32 row4 = b.template Shuffle<{{1,2,0}}>(); - - VectorF32 row3 = a.template Shuffle<{{2,0,1}}>(); - VectorF32 row2 = b.template Shuffle<{{2,0,1}}>(); - - VectorF32 result = row3 * row4; - return VectorF32::MulitplySub(row1, row2, result); - } - - template ShuffleValues> - constexpr VectorF32 Shuffle() { - if constexpr(VectorBase::template CheckEpi32Shuffle()) { - constexpr std::uint8_t imm = VectorBase::template GetShuffleMaskEpi32(); - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32(_mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(this->v), imm))); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32(_mm256_castsi256_ps(_mm256_shuffle_epi32(_mm256_castps_si256(this->v), imm))); - #ifdef __AVX512F__ - } else { - return VectorF32(_mm512_castsi512_ps(_mm512_shuffle_epi32(_mm512_castps_si512(this->v), imm))); - #endif - } - } else if constexpr(VectorBase::template CheckEpi8Shuffle()) { - constexpr std::array::Alignment> shuffleMask = VectorBase::template GetShuffleMaskEpi8(); - if constexpr(std::is_same_v::VectorType, __m128>) { - __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); - return VectorF32(_mm_castsi128_ps(_mm_shuffle_epi8(_mm_castps_si128(this->v), shuffleVec))); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - #ifdef __AVX512BW__ - __m256i shuffleVec = _mm256_loadu_si256(reinterpret_cast(shuffleMask.data())); - return VectorF32(_mm256_castsi256_ps( _mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castps_si256(this->v)),_mm512_castsi256_si512(shuffleVec))))); - #else - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetPermuteMaskEpi32(); - __m256i permIdx = _mm256_loadu_si256(reinterpret_cast(permMask.data())); - return VectorF32(_mm256_castsi256_ps(_mm256_permutevar8x32_epi32(_mm256_castps_si256(this->v), permIdx))); - #endif - #ifdef __AVX512F__ - } else { - __m512i shuffleVec = _mm512_loadu_si512(reinterpret_cast(shuffleMask.data())); - return VectorF32(_mm512_castsi512_ps(_mm512_shuffle_epi8(_mm512_castps_si512(this->v), shuffleVec))); - #endif - } - } else { - if constexpr(std::is_same_v::VectorType, __m128>) { - constexpr std::array::Alignment> shuffleMask = VectorBase::template GetShuffleMaskEpi8(); - __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); - return VectorF32(_mm_castsi128_ps(_mm_shuffle_epi8(_mm_castps_si128(this->v), shuffleVec))); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetPermuteMaskEpi32(); - __m256i permIdx = _mm256_loadu_si256(reinterpret_cast(permMask.data())); - return VectorF32(_mm256_castsi256_ps(_mm256_permutevar8x32_epi32(_mm256_castps_si256(this->v), permIdx))); - #ifdef __AVX512F__ - } else { - constexpr std::array::AlignmentElement> permMask = VectorBase::template GetPermuteMaskEpi32(); - __m512i permIdx = _mm512_loadu_epi32(permMask.data()); - return VectorF32(_mm512_castsi512_ps(_mm512_permutexvar_epi32(permIdx, _mm512_castps_si512(this->v)))); - #endif - } - } - } - - // Public variadic surface — one name per op, arity locked to BatchSize. - // The Pack helpers below carry the SIMD bodies and the per-(Len,Packing) - // requires clauses; this wrapper just forwards once arity matches. - template - requires ((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto Normalize(VectorF32 first, Rest... rest) { - return NormalizePack(first, rest...); - } - - template - requires ((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto Length(VectorF32 first, Rest... rest) { - return LengthPack(first, rest...); - } - - template - requires ((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto LengthSq(VectorF32 first, Rest... rest) { - return LengthSqPack(first, rest...); - } - - template - requires ((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == 2 * VectorBase::BatchSize)) - constexpr static auto Dot(VectorF32 first, Rest... rest) { - return DotPack(first, rest...); - } - - private: - constexpr static std::array, VectorBase::BatchSize> NormalizePack( - VectorF32 A, - VectorF32 B, - VectorF32 C, - VectorF32 D - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - if constexpr(std::is_same_v::VectorType, __m128>) { - VectorF32<1, 4> lenght = LengthNoShuffle(A, C, B, D); - constexpr float oneArr[] {1, 1, 1, 1}; - __m128 one = _mm_loadu_ps(oneArr); - VectorF32<4, 1> fLenght(_mm_div_ps(one, lenght.v)); - - VectorF32<4, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0}}>(); - VectorF32<4, 1> fLenghtB = fLenght.template Shuffle<{{1,1,1,1}}>(); - VectorF32<4, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2}}>(); - VectorF32<4, 1> fLenghtD = fLenght.template Shuffle<{{3,3,3,3}}>(); - - return { - _mm_mul_ps(A.v, fLenghtA.v), - _mm_mul_ps(B.v, fLenghtB.v), - _mm_mul_ps(C.v, fLenghtC.v), - _mm_mul_ps(D.v, fLenghtD.v) - }; - } else if constexpr(std::is_same_v::VectorType, __m256>) { - VectorF32<1, 8> lenght = LengthNoShuffle(A, C, B, D); - constexpr float oneArr[] {1, 1, 1, 1, 1, 1, 1, 1}; - __m256 one = _mm256_loadu_ps(oneArr); - VectorF32<8, 1> fLenght(_mm256_div_ps(one, lenght.v)); - - - VectorF32<8, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,4,4,4,4}}>(); - VectorF32<8, 1> fLenghtB = fLenght.template Shuffle<{{1,1,1,1,5,5,5,5}}>(); - VectorF32<8, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,6,6,6,6}}>(); - VectorF32<8, 1> fLenghtD = fLenght.template Shuffle<{{3,3,3,3,7,7,7,7}}>(); - - return { - _mm256_mul_ps(A.v, fLenghtA.v), - _mm256_mul_ps(B.v, fLenghtB.v), - _mm256_mul_ps(C.v, fLenghtC.v), - _mm256_mul_ps(D.v, fLenghtD.v) - }; - #if defined(__AVX512F__) - } else { - VectorF32<1, 16> lenght = LengthNoShuffle(A, C, B, D); - constexpr float oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; - __m512 one = _mm512_loadu_ps(oneArr); - VectorF32<16, 1> fLenght(_mm512_div_ps(one, lenght.v)); - VectorF32<16, 1> fLenght2(lenght.v); - - VectorF32<16, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,4,4,4,4,8,8,8,8,12,12,12,12}}>(); - VectorF32<16, 1> fLenghtB = fLenght.template Shuffle<{{1,1,1,1,5,5,5,5,9,9,9,9,13,13,13,13}}>(); - VectorF32<16, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,6,6,6,6,10,10,10,10,14,14,14,14}}>(); - VectorF32<16, 1> fLenghtD = fLenght.template Shuffle<{{3,3,3,3,7,7,7,7,11,11,11,11,15,15,15,15}}>(); - - - return { - VectorF32(_mm512_mul_ps(A.v, fLenghtA.v)), - VectorF32(_mm512_mul_ps(B.v, fLenghtB.v)), - VectorF32(_mm512_mul_ps(C.v, fLenghtC.v)), - VectorF32(_mm512_mul_ps(D.v, fLenghtD.v)), - }; - #endif - } - } - - constexpr static std::array, VectorBase::BatchSize> NormalizePack( - VectorF32 A, - VectorF32 B, - VectorF32 C, - VectorF32 D - ) requires(Len == 3 && Packing == 1) { - VectorF32<1, 4> lenght = Length(A, B, C, D); - constexpr float oneArr[] {1, 1, 1, 1}; - __m128 one = _mm_loadu_ps(oneArr); - VectorF32<4, 1> fLenght(_mm_div_ps(one, lenght.v)); - - VectorF32<4, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0}}>(); - VectorF32<4, 1> fLenghtB = fLenght.template Shuffle<{{1,1,1,1}}>(); - VectorF32<4, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2}}>(); - VectorF32<4, 1> fLenghtD = fLenght.template Shuffle<{{3,3,3,3}}>(); - - return { - _mm_mul_ps(A.v, fLenghtA.v), - _mm_mul_ps(B.v, fLenghtB.v), - _mm_mul_ps(C.v, fLenghtC.v), - _mm_mul_ps(D.v, fLenghtD.v) - }; - } - - constexpr static std::array, VectorBase::BatchSize> NormalizePack( - VectorF32 A, - VectorF32 B, - VectorF32 C, - VectorF32 D - ) requires(Len == 3 && Packing == 2) { - VectorF32<1, 8> lenght = Length(A, B, C, D); - constexpr float oneArr[] {1, 1, 1, 1, 1, 1, 1, 1}; - __m256 one = _mm256_loadu_ps(oneArr); - VectorF32<8, 1> fLenght(_mm256_div_ps(one, lenght.v)); - - VectorF32<8, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0, 1,1,1}}>(); - VectorF32<8, 1> fLenghtB = fLenght.template Shuffle<{{2,2,2, 3,3,3}}>(); - VectorF32<8, 1> fLenghtC = fLenght.template Shuffle<{{4,4,4, 5,5,5}}>(); - VectorF32<8, 1> fLenghtD = fLenght.template Shuffle<{{6,6,6, 7,7,7}}>(); - - return { - _mm256_mul_ps(A.v, fLenghtA.v), - _mm256_mul_ps(B.v, fLenghtB.v), - _mm256_mul_ps(C.v, fLenghtC.v), - _mm256_mul_ps(D.v, fLenghtD.v) - }; - } - - #ifdef __AVX512F__ - constexpr static std::array, VectorBase::BatchSize> NormalizePack( - VectorF32 A, - VectorF32 B, - VectorF32 C - ) requires(Len == 3 && Packing == 5) { - VectorF32<1, 15> lenght = Length(A, B, C); - constexpr float oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; - __m512 one = _mm512_loadu_ps(oneArr); - VectorF32<15, 1> fLenght(_mm512_div_ps(one, lenght.v)); - - VectorF32<15, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0, 1,1,1, 2,2,2, 3,3,3, 4,4,4}}>(); - VectorF32<15, 1> fLenghtB = fLenght.template Shuffle<{{5,5,5, 6,6,6, 7,7,7, 8,8,8, 9,9,9}}>(); - VectorF32<15, 1> fLenghtC = fLenght.template Shuffle<{{10,10,10, 11,11,11, 12,12,12, 13,13,13, 14,14,14}}>(); - - return { - _mm512_mul_ps(A.v, fLenghtA.v), - _mm512_mul_ps(B.v, fLenghtB.v), - _mm512_mul_ps(C.v, fLenghtC.v), - }; - } - #endif - - constexpr static std::array, VectorBase::BatchSize> NormalizePack( - VectorF32 A, - VectorF32 B - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - if constexpr(std::is_same_v::VectorType, __m128>) { - VectorF32<1, 4> lenght = LengthNoShuffle(A, B); - constexpr float oneArr[] {1, 1, 1, 1}; - __m128 one = _mm_loadu_ps(oneArr); - VectorF32<4, 1> fLenght(_mm_div_ps(one, lenght.v)); - - VectorF32<4, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1}}>(); - VectorF32<4, 1> fLenghtB = fLenght.template Shuffle<{{2,2,3,3}}>(); - - return { - _mm_mul_ps(A.v, fLenghtA.v), - _mm_mul_ps(B.v, fLenghtB.v), - }; - } else if constexpr(std::is_same_v::VectorType, __m256>) { - VectorF32<1, 8> lenght = LengthNoShuffle(A, B); - constexpr float oneArr[] {1, 1, 1, 1, 1, 1, 1, 1}; - __m256 one = _mm256_loadu_ps(oneArr); - VectorF32<8, 1> fLenght(_mm256_div_ps(one, lenght.v)); - - VectorF32<8, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,4,4,5,5}}>(); - VectorF32<8, 1> fLenghtB = fLenght.template Shuffle<{{2,2,3,3,6,6,7,7}}>(); - - return { - _mm256_mul_ps(A.v, fLenghtA.v), - _mm256_mul_ps(B.v, fLenghtB.v), - }; - #ifdef __AVX512F__ - } else { - VectorF32<1, 16> lenght = LengthNoShuffle(A, B); - constexpr float oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; - __m512 one = _mm512_loadu_ps(oneArr); - VectorF32<16, 1> fLenght(_mm512_div_ps(one, lenght.v)); - - VectorF32<16, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,4,4,5,5,8,8,9,9,12,12,13,13}}>(); - VectorF32<16, 1> fLenghtB = fLenght.template Shuffle<{{2,2,3,3,6,6,7,7,10,10,11,11,14,14,15,15}}>(); - - return { - _mm512_mul_ps(A.v, fLenghtA.v), - _mm512_mul_ps(B.v, fLenghtB.v), - }; - #endif - } - } - - constexpr static VectorF32<1, Packing*4> LengthPack( - VectorF32 A, - VectorF32 B, - VectorF32 C, - VectorF32 D - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - VectorF32<1, Packing*4> lenghtSq = LengthSqPack(A, B, C, D); - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32<1, Packing*4>(_mm_sqrt_ps(lenghtSq.v)); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32<1, Packing*4>(_mm256_sqrt_ps(lenghtSq.v)); - } else { - return VectorF32<1, Packing*4>(_mm512_sqrt_ps(lenghtSq.v)); - } - } - - constexpr static VectorF32<1, 4> LengthPack( - VectorF32 A, - VectorF32 B, - VectorF32 C, - VectorF32 D - ) requires(Len == 3 && Packing == 1) { - VectorF32<1, 4> lenghtSq = LengthSqPack(A, B, C, D); - return VectorF32<1, 4>(_mm_sqrt_ps(lenghtSq.v)); - } - - constexpr static VectorF32<1, 8> LengthPack( - VectorF32 A, - VectorF32 B, - VectorF32 C, - VectorF32 D - ) requires(Len == 3 && Packing == 2) { - VectorF32<1, 8> lenghtSq = LengthSqPack(A, B, C, D); - return VectorF32<1, Packing*4>(_mm256_sqrt_ps(lenghtSq.v)); - } - - #ifdef __AVX512F__ - constexpr static VectorF32<1, 15> LengthPack( - VectorF32 A, - VectorF32 B, - VectorF32 C - ) requires(Len == 3 && Packing == 5) { - VectorF32<1, 15> lenghtSq = LengthSqPack(A, B, C); - return VectorF32<1, 15>(_mm512_sqrt_ps(lenghtSq.v)); - } - #endif - - constexpr static VectorF32<1, Packing*2> LengthPack( - VectorF32 A, - VectorF32 C - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - VectorF32<1, Packing*2> lenghtSq = LengthSqPack(A, C); - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32<1, Packing*2>(_mm_sqrt_ps(lenghtSq.v)); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32<1, Packing*2>(_mm256_sqrt_ps(lenghtSq.v)); - #ifdef __AVX512F__ - } else { - return VectorF32<1, Packing*2>(_mm512_sqrt_ps(lenghtSq.v)); - #endif - } - } - - constexpr static VectorF32<1, Packing*4> LengthSqPack( - VectorF32 A, - VectorF32 B, - VectorF32 C, - VectorF32 D - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - return DotPack(A, A, B, B, C, C, D, D); - } - - constexpr static VectorF32<1, 4> LengthSqPack( - VectorF32 A, - VectorF32 B, - VectorF32 C, - VectorF32 D - ) requires(Len == 3 && Packing == 1) { - return DotPack(A, A, B, B, C, C, D, D); - } - - constexpr static VectorF32<1, 8> LengthSqPack( - VectorF32 A, - VectorF32 B, - VectorF32 C, - VectorF32 D - ) requires(Len == 3 && Packing == 2) { - return DotPack(A, A, B, B, C, C, D, D); - } - - #ifdef __AVX512F__ - constexpr static VectorF32<1, 15> LengthSqPack( - VectorF32 A, - VectorF32 B, - VectorF32 C - ) requires(Len == 3 && Packing == 5) { - return DotPack(A, A, B, B, C, C); - } - #endif - - constexpr static VectorF32<1, Packing*2> LengthSqPack( - VectorF32 A, - VectorF32 C - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - return DotPack(A, A, C, C); - } - - constexpr static VectorF32<1, Packing*4> DotPack( - VectorF32 A0, VectorF32 A1, - VectorF32 B0, VectorF32 B1, - VectorF32 C0, VectorF32 C1, - VectorF32 D0, VectorF32 D1 - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - if constexpr(std::is_same_v::VectorType, __m128>) { - return DotNoShuffle(A0, A1, C0, C1, B0, B1, D0, D1); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - VectorF32<8, 1> vec(DotNoShuffle(A0, A1, B0, B1, C0, C1, D0, D1).v); - vec = vec.template Shuffle<{{ - 0,4,2,6, - 1,5,3,7, - }}>(); - return vec.v; - #ifdef __AVX512F__ - } else { - VectorF32<16, 1> vec(DotNoShuffle(A0, A1, B0, B1, C0, C1, D0, D1).v); - vec = vec.template Shuffle<{{ - 0,4,8,12, - 2,6,10,14, - 1,5,9,13, - 3,7,11,15 - }}>(); - return vec.v; - #endif - } - } - - constexpr static VectorF32<1, 4> DotPack( - VectorF32 A0, VectorF32 A1, - VectorF32 B0, VectorF32 B1, - VectorF32 C0, VectorF32 C1, - VectorF32 D0, VectorF32 D1 - ) requires(Len == 3 && Packing == 1) { - // Each register: [X1 X2 X3 _] - // 4 pairs (A,B,C,D) → 4 dot products → 1 x __m128 - // - // After element-wise multiply: - // mulA = [a1 a2 a3 _] (where ai = A0[i]*A1[i]) - // mulB = [b1 b2 b3 _] - // mulC = [c1 c2 c3 _] - // mulD = [d1 d2 d3 _] - // - // We need: result = [a1+a2+a3, b1+b2+b3, c1+c2+c3, d1+d2+d3] - // - // Transpose to get: - // row1 = [a1 b1 c1 d1] - // row2 = [a2 b2 c2 d2] - // row3 = [a3 b3 c3 d3] - // Then sum rows. - - __m128 mulA = _mm_mul_ps(A0.v, A1.v); - __m128 mulB = _mm_mul_ps(B0.v, B1.v); - __m128 mulC = _mm_mul_ps(C0.v, C1.v); - __m128 mulD = _mm_mul_ps(D0.v, D1.v); - - // Standard 4x4 transpose (only first 3 rows matter, 4th is garbage) - // unpacklo/hi interleave pairs of 32-bit elements - __m128 tmp0 = _mm_unpacklo_ps(mulA, mulB); // a1 b1 a2 b2 - __m128 tmp1 = _mm_unpackhi_ps(mulA, mulB); // a3 b3 _ _ - __m128 tmp2 = _mm_unpacklo_ps(mulC, mulD); // c1 d1 c2 d2 - __m128 tmp3 = _mm_unpackhi_ps(mulC, mulD); // c3 d3 _ _ - - __m128 row1 = _mm_movelh_ps(tmp0, tmp2); // a1 b1 c1 d1 - __m128 row2 = _mm_movehl_ps(tmp2, tmp0); // a2 b2 c2 d2 - __m128 row3 = _mm_movelh_ps(tmp1, tmp3); // a3 b3 c3 d3 - - row1 = _mm_add_ps(row1, row2); - row1 = _mm_add_ps(row1, row3); - - return row1; - } - - constexpr static VectorF32<1, 8> DotPack( - VectorF32 A0, VectorF32 A1, - VectorF32 B0, VectorF32 B1, - VectorF32 C0, VectorF32 C1, - VectorF32 D0, VectorF32 D1 - ) requires(Len == 3 && Packing == 2) { - // Each register: [X1 X2 X3 Y1 Y2 Y3 _ _] - // 4 pairs × 2 vectors each = 8 dot products → 1 x __m256 - // - // After multiply: - // mulA = [a1 a2 a3 b1 b2 b3 _ _] - // mulB = [c1 c2 c3 d1 d2 d3 _ _] - // mulC = [e1 e2 e3 f1 f2 f3 _ _] - // mulD = [g1 g2 g3 h1 h2 h3 _ _] - // - // We need result = [a·, b·, c·, d·, e·, f·, g·, h·] - // where x· = x1+x2+x3 - // - // Strategy: use permute to gather element 1s, 2s, 3s across all 8 vectors, - // then add. - // - // Gather indices (from the concatenated view of mulA|mulB|mulC|mulD): - // vec: a a a b b b _ _ c c c d d d _ _ e e e f f f _ _ g g g h h h _ _ - // idx: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 - // - // elem1 = [a1, b1, c1, d1, e1, f1, g1, h1] → indices [0, 3, 8, 11, 16, 19, 24, 27] - // elem2 = [a2, b2, c2, d2, e2, f2, g2, h2] → indices [1, 4, 9, 12, 17, 20, 25, 28] - // elem3 = [a3, b3, c3, d3, e3, f3, g3, h3] → indices [2, 5, 10, 13, 18, 21, 26, 29] - // - // Unfortunately AVX2 doesn't have cross-register permutes for 8x32 easily. - // Use vpermd (_mm256_permutevar8x32) within pairs, then blend/combine. - // - // Within each 256-bit register [X1 X2 X3 Y1 Y2 Y3 _ _]: - // elem1_local = [X1 Y1 ...] → gather from indices 0,3 - // elem2_local = [X2 Y2 ...] → gather from indices 1,4 - // elem3_local = [X3 Y3 ...] → gather from indices 2,5 - // - // After permutevar8x32 on each mul register: - // From mulA: row1_part = [a1 b1 _ _ _ _ _ _] - // From mulB: row1_part = [c1 d1 _ _ _ _ _ _] - // From mulC: row1_part = [e1 f1 _ _ _ _ _ _] - // From mulD: row1_part = [g1 h1 _ _ _ _ _ _] - // - // Then combine with unpack/shuffle to get full rows. - - __m256 mulA = _mm256_mul_ps(A0.v, A1.v); // a1 a2 a3 b1 b2 b3 _ _ - __m256 mulB = _mm256_mul_ps(B0.v, B1.v); // c1 c2 c3 d1 d2 d3 _ _ - __m256 mulC = _mm256_mul_ps(C0.v, C1.v); // e1 e2 e3 f1 f2 f3 _ _ - __m256 mulD = _mm256_mul_ps(D0.v, D1.v); // g1 g2 g3 h1 h2 h3 _ _ - - // Permute each register to gather elements by position. - // For each register [X1 X2 X3 Y1 Y2 Y3 U U]: - // perm1: [X1 Y1 X2 Y2 X3 Y3 _ _] → indices {0,3,1,4,2,5,6,7} - __m256i permIdx = _mm256_setr_epi32(0, 3, 1, 4, 2, 5, 6, 7); - - // After permute: [X1 Y1 X2 Y2 X3 Y3 _ _] - __m256 pA = _mm256_permutevar8x32_ps(mulA, permIdx); // a1 b1 a2 b2 a3 b3 _ _ - __m256 pB = _mm256_permutevar8x32_ps(mulB, permIdx); // c1 d1 c2 d2 c3 d3 _ _ - __m256 pC = _mm256_permutevar8x32_ps(mulC, permIdx); // e1 f1 e2 f2 e3 f3 _ _ - __m256 pD = _mm256_permutevar8x32_ps(mulD, permIdx); // g1 h1 g2 h2 g3 h3 _ _ - - // Now combine pairs. Each pair contributes 4 consecutive results. - // pA has [a1 b1 a2 b2 a3 b3 _ _], pB has [c1 d1 c2 d2 c3 d3 _ _] - // We want: - // row1 = [a1 b1 c1 d1 | e1 f1 g1 h1] - // row2 = [a2 b2 c2 d2 | e2 f2 g2 h2] - // row3 = [a3 b3 c3 d3 | e3 f3 g3 h3] - // - // From pA: elements at [0,1] are elem1, [2,3] are elem2, [4,5] are elem3 - // From pB: elements at [0,1] are elem1, [2,3] are elem2, [4,5] are elem3 - // - // Use unpacklo_epi64 to interleave 64-bit chunks: - // unpacklo64(pA, pB) within 128-bit lanes: - // lo lane: pA[0:1]=a1,b1 | pB[0:1]=c1,d1 → [a1 b1 c1 d1] - // hi lane: pA[4:5]=a3,b3 | pB[4:5]=c3,d3 → [a3 b3 c3 d3] - // → [a1 b1 c1 d1 | a3 b3 c3 d3] - // - // unpackhi64(pA, pB) within 128-bit lanes: - // lo lane: pA[2:3]=a2,b2 | pB[2:3]=c2,d2 → [a2 b2 c2 d2] - // hi lane: pA[6:7]=_,_ | pB[6:7]=_,_ → garbage - // → [a2 b2 c2 d2 | _ _ _ _] - - __m256i AB_lo = _mm256_unpacklo_epi64( - _mm256_castps_si256(pA), _mm256_castps_si256(pB)); // [a1 b1 c1 d1 | a3 b3 c3 d3] - __m256i AB_hi = _mm256_unpackhi_epi64( - _mm256_castps_si256(pA), _mm256_castps_si256(pB)); // [a2 b2 c2 d2 | _ _ _ _] - - __m256i CD_lo = _mm256_unpacklo_epi64( - _mm256_castps_si256(pC), _mm256_castps_si256(pD)); // [e1 f1 g1 h1 | e3 f3 g3 h3] - __m256i CD_hi = _mm256_unpackhi_epi64( - _mm256_castps_si256(pC), _mm256_castps_si256(pD)); // [e2 f2 g2 h2 | _ _ _ _] - - // row1 = [a1 b1 c1 d1 | e1 f1 g1 h1] → lo 128 of AB_lo, lo 128 of CD_lo - // row2 = [a2 b2 c2 d2 | e2 f2 g2 h2] → lo 128 of AB_hi, lo 128 of CD_hi - // row3 = [a3 b3 c3 d3 | e3 f3 g3 h3] → hi 128 of AB_lo, hi 128 of CD_lo - - __m256 row1 = _mm256_castsi256_ps(_mm256_permute2x128_si256(AB_lo, CD_lo, 0x20)); // lo,lo - __m256 row2 = _mm256_castsi256_ps(_mm256_permute2x128_si256(AB_hi, CD_hi, 0x20)); // lo,lo - __m256 row3 = _mm256_castsi256_ps(_mm256_permute2x128_si256(AB_lo, CD_lo, 0x31)); // hi,hi - - row1 = _mm256_add_ps(row1, row2); - row1 = _mm256_add_ps(row1, row3); - - return row1; - } - - #ifdef __AVX512F__ - constexpr static VectorF32<1, 15> DotPack( - VectorF32 A0, VectorF32 A1, - VectorF32 B0, VectorF32 B1, - VectorF32 C0, VectorF32 C1 - ) requires(Len == 3 && Packing == 5) { - // __m512: Each register: [A1 A2 A3 B1 B2 B3 C1 C2 C3 D1 D2 D3 E1 E2 E3 _] - // 3 pairs × 5 vectors each = 15 dot products → fits in 1 x __m512 (slot 16 unused) - // - // After multiply of 3 pairs: - // mul0 = [a1 a2 a3 b1 b2 b3 c1 c2 c3 d1 d2 d3 e1 e2 e3 _] - // mul1 = [f1 f2 f3 g1 g2 g3 h1 h2 h3 i1 i2 i3 j1 j2 j3 _] - // mul2 = [k1 k2 k3 l1 l2 l3 m1 m2 m3 n1 n2 n3 o1 o2 o3 _] - // - // Result = [a· b· c· d· e· f· g· h· i· j· k· l· m· n· o· _] - // - // Strategy: for each mul register, gather element 1s, 2s, 3s with vpermps, - // then combine across registers. - // - // From mul0: 5 vectors at positions {0,1,2}, {3,4,5}, {6,7,8}, {9,10,11}, {12,13,14} - // elem1 = indices {0, 3, 6, 9, 12} → positions 0..4 of result - // elem2 = indices {1, 4, 7, 10, 13} - // elem3 = indices {2, 5, 8, 11, 14} - - __m512 mul0 = _mm512_mul_ps(A0.v, A1.v); - __m512 mul1 = _mm512_mul_ps(B0.v, B1.v); - __m512 mul2 = _mm512_mul_ps(C0.v, C1.v); - - // Gather elem1, elem2, elem3 from each mul register - // Each register has 5 vec3s: extract element 1,2,3 of each into consecutive positions - __m512i idx1 = _mm512_setr_epi32(0, 3, 6, 9, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - __m512i idx2 = _mm512_setr_epi32(1, 4, 7, 10, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - __m512i idx3 = _mm512_setr_epi32(2, 5, 8, 11, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); - - // From mul0 → results 0..4, from mul1 → results 5..9, from mul2 → results 10..14 - // Gather from each, then combine. - - __m512 e1_0 = _mm512_permutexvar_ps(idx1, mul0); // [a1 b1 c1 d1 e1 ...] - __m512 e2_0 = _mm512_permutexvar_ps(idx2, mul0); // [a2 b2 c2 d2 e2 ...] - __m512 e3_0 = _mm512_permutexvar_ps(idx3, mul0); // [a3 b3 c3 d3 e3 ...] - - __m512 e1_1 = _mm512_permutexvar_ps(idx1, mul1); // [f1 g1 h1 i1 j1 ...] - __m512 e2_1 = _mm512_permutexvar_ps(idx2, mul1); // [f2 g2 h2 i2 j2 ...] - __m512 e3_1 = _mm512_permutexvar_ps(idx3, mul1); // [f3 g3 h3 i3 j3 ...] - - __m512 e1_2 = _mm512_permutexvar_ps(idx1, mul2); // [k1 l1 m1 n1 o1 ...] - __m512 e2_2 = _mm512_permutexvar_ps(idx2, mul2); // [k2 l2 m2 n2 o2 ...] - __m512 e3_2 = _mm512_permutexvar_ps(idx3, mul2); // [k3 l3 m3 n3 o3 ...] - - // Now combine: we need positions 0..4 from reg0, 5..9 from reg1, 10..14 from reg2 - // Use masked moves to assemble the final row vectors. - // mask for positions 0-4: 0b0000000000011111 = 0x001F - // mask for positions 5-9: 0b0000001111100000 = 0x03E0 - // mask for positions 10-14: 0b0111110000000000 = 0x7C00 - - // For reg1, its results are in positions 0..4 but need to go to 5..9. - // For reg2, its results are in positions 0..4 but need to go to 10..14. - // Use a different approach: permute reg1/reg2 results to their target positions. - - // Shift reg1 results from slots 0..4 to slots 5..9 - __m512i shiftIdx1 = _mm512_setr_epi32(0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 0, 0, 0, 0, 0, 0); - // Shift reg2 results from slots 0..4 to slots 10..14 - __m512i shiftIdx2 = _mm512_setr_epi32(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 0); - - __m512 e1_1_shifted = _mm512_permutexvar_ps(shiftIdx1, e1_1); - __m512 e2_1_shifted = _mm512_permutexvar_ps(shiftIdx1, e2_1); - __m512 e3_1_shifted = _mm512_permutexvar_ps(shiftIdx1, e3_1); - - __m512 e1_2_shifted = _mm512_permutexvar_ps(shiftIdx2, e1_2); - __m512 e2_2_shifted = _mm512_permutexvar_ps(shiftIdx2, e2_2); - __m512 e3_2_shifted = _mm512_permutexvar_ps(shiftIdx2, e3_2); - - // Blend: take positions 0..4 from reg0, 5..9 from reg1, 10..14 from reg2 - __mmask16 mask_5_9 = 0x03E0u; // bits 5-9 - __mmask16 mask_10_14 = 0x7C00u; // bits 10-14 - - __m512 row1 = _mm512_mask_mov_ps(e1_0, mask_5_9, e1_1_shifted); - row1 = _mm512_mask_mov_ps(row1, mask_10_14, e1_2_shifted); - - __m512 row2 = _mm512_mask_mov_ps(e2_0, mask_5_9, e2_1_shifted); - row2 = _mm512_mask_mov_ps(row2, mask_10_14, e2_2_shifted); - - __m512 row3 = _mm512_mask_mov_ps(e3_0, mask_5_9, e3_1_shifted); - row3 = _mm512_mask_mov_ps(row3, mask_10_14, e3_2_shifted); - - row1 = _mm512_add_ps(row1, row2); - row1 = _mm512_add_ps(row1, row3); - - return row1; - } - #endif - - constexpr static VectorF32<1, Packing*2> DotPack( - VectorF32 A0, VectorF32 A1, - VectorF32 C0, VectorF32 C1 - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - if constexpr(std::is_same_v::VectorType, __m128>) { - return DotNoShuffle(A0, A1, C0, C1); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - VectorF32<8, 1> vec(DotNoShuffle(A0, A1, C0, C1).v); - vec = vec.template Shuffle<{{ - 0,1, 4,5, - 2,3, 6,7, - }}>(); - return vec.v; - #ifdef __AVX512F__ - } else { - VectorF32<16, 1> vec(DotNoShuffle(A0, A1, C0, C1).v); - vec = vec.template Shuffle<{{ - 0,1, 4,5, - 8,9, 12,13, - 2,3, 6,7, - 10,11, 14,15 - }}>(); - return vec.v; - #endif - } - } - - - private: - constexpr static VectorF32<1, Packing*4> LengthNoShuffle( - VectorF32 A, - VectorF32 B, - VectorF32 C, - VectorF32 D - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - VectorF32<1, Packing*4> lenghtSq = LengthSqNoShuffle(A, B, C, D); - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32<1, Packing*4>(_mm_sqrt_ps(lenghtSq.v)); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32<1, Packing*4>(_mm256_sqrt_ps(lenghtSq.v)); - #ifdef __AVX512F__ - } else { - return VectorF32<1, Packing*4>(_mm512_sqrt_ps(lenghtSq.v)); - #endif - } - } - - constexpr static VectorF32<1, Packing*2> LengthNoShuffle( - VectorF32 A, - VectorF32 C - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - VectorF32<1, Packing*2> lenghtSq = LengthSqNoShuffle(A, C); - if constexpr(std::is_same_v::VectorType, __m128>) { - return VectorF32<1, Packing*2>(_mm_sqrt_ps(lenghtSq.v)); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - return VectorF32<1, Packing*2>(_mm256_sqrt_ps(lenghtSq.v)); - #ifdef __AVX512F__ - } else { - return VectorF32<1, Packing*2>(_mm512_sqrt_ps(lenghtSq.v)); - #endif - } - } - - constexpr static VectorF32<1, Packing*4> LengthSqNoShuffle( - VectorF32 A, - VectorF32 B, - VectorF32 C, - VectorF32 D - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - return DotNoShuffle(A, A, B, B, C, C, D, D); - } - - constexpr static VectorF32<1, Packing*2> LengthSqNoShuffle( - VectorF32 A, - VectorF32 C - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - return DotNoShuffle(A, A, C, C); - } - - - constexpr static VectorF32<1, Packing*4> DotNoShuffle( - VectorF32 A0, VectorF32 A1, - VectorF32 B0, VectorF32 B1, - VectorF32 C0, VectorF32 C1, - VectorF32 D0, VectorF32 D1 - ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { - if constexpr(std::is_same_v::VectorType, __m128>) { - __m128 mulA = _mm_mul_ps(A0.v, A1.v); - __m128 mulB = _mm_mul_ps(B0.v, B1.v); - - __m128i row12Temp1 = _mm_unpacklo_epi32(_mm_castps_si128(mulA), _mm_castps_si128(mulB)); // A1 B1 A2 B2 - __m128i row34Temp1 = _mm_unpackhi_epi32(_mm_castps_si128(mulA), _mm_castps_si128(mulB)); // A3 B3 A4 B4 - - __m128 mulC = _mm_mul_ps(C0.v, C1.v); - __m128 mulD = _mm_mul_ps(D0.v, D1.v); - - __m128i row12Temp2 = _mm_unpacklo_epi32(_mm_castps_si128(mulC), _mm_castps_si128(mulD)); // C1 D1 C2 D2 - __m128i row34Temp2 = _mm_unpackhi_epi32(_mm_castps_si128(mulC), _mm_castps_si128(mulD)); // C3 D3 C4 D4 - - __m128 row1 = _mm_unpacklo_epi32(row12Temp1, row12Temp2); // A1 C1 B1 D1 - __m128 row2 = _mm_unpackhi_epi32(row12Temp1, row12Temp2); // A2 C2 B2 D2 - __m128 row3 = _mm_unpacklo_epi32(row34Temp1, row34Temp2); // A3 C3 B3 D3 - __m128 row4 = _mm_unpackhi_epi32(row34Temp1, row34Temp2); // A4 C4 B4 D4 - - - row1 = _mm_add_ps(row1, row2); - row1 = _mm_add_ps(row1, row3); - row1 = _mm_add_ps(row1, row4); - - return row1; - } else if constexpr(std::is_same_v::VectorType, __m256>) { - __m256 mulA = _mm256_mul_ps(A0.v, A1.v); - __m256 mulB = _mm256_mul_ps(B0.v, B1.v); - - __m256i row12Temp1 = _mm256_unpacklo_epi32(_mm256_castps_si256(mulA), _mm256_castps_si256(mulB)); // A1 B1 A2 B2 - __m256i row34Temp1 = _mm256_unpackhi_epi32(_mm256_castps_si256(mulA), _mm256_castps_si256(mulB)); // A3 B3 A4 B4 - - __m256 mulC = _mm256_mul_ps(C0.v, C1.v); - __m256 mulD = _mm256_mul_ps(D0.v, D1.v); - - __m256i row12Temp2 = _mm256_unpacklo_epi32(_mm256_castps_si256(mulC), _mm256_castps_si256(mulD)); // C1 D1 C2 D2 - __m256i row34Temp2 = _mm256_unpackhi_epi32(_mm256_castps_si256(mulC), _mm256_castps_si256(mulD)); // C3 D3 C4 D4 - - __m256 row1 = _mm256_unpacklo_epi32(row12Temp1, row12Temp2); // A1 C1 B1 D1 - __m256 row2 = _mm256_unpackhi_epi32(row12Temp1, row12Temp2); //A2 C2 B2 D2 - __m256 row3 = _mm256_unpacklo_epi32(row34Temp1, row34Temp2); // A3 C3 B3 D3 - __m256 row4 = _mm256_unpackhi_epi32(row34Temp1, row34Temp2); // A4 C4 B4 D4 - - row1 = _mm256_add_ps(row1, row2); - row1 = _mm256_add_ps(row1, row3); - row1 = _mm256_add_ps(row1, row4); - - return row1; - #ifdef __AVX512F__ - } else { - __m512 mulA = _mm512_mul_ps(A0.v, A1.v); - __m512 mulB = _mm512_mul_ps(B0.v, B1.v); - - __m512i row12Temp1 = _mm512_unpacklo_epi32(_mm512_castps_si512(mulA), _mm512_castps_si512(mulB)); // A1 B1 A2 B2 - __m512i row34Temp1 = _mm512_unpackhi_epi32(_mm512_castps_si512(mulA), _mm512_castps_si512(mulB)); // A3 B3 A4 B4 - - __m512 mulC = _mm512_mul_ps(C0.v, C1.v); - __m512 mulD = _mm512_mul_ps(D0.v, D1.v); - - __m512i row12Temp2 = _mm512_unpacklo_epi32(_mm512_castps_si512(mulC), _mm512_castps_si512(mulD)); // C1 D1 C2 D2 - __m512i row34Temp2 = _mm512_unpackhi_epi32(_mm512_castps_si512(mulC), _mm512_castps_si512(mulD)); // C3 D3 C4 D4 - - __m512 row1 = _mm512_unpacklo_epi32(row12Temp1, row12Temp2); // A1 C1 B1 D1 - __m512 row2 = _mm512_unpackhi_epi32(row12Temp1, row12Temp2); //A2 C2 B2 D2 - __m512 row3 = _mm512_unpacklo_epi32(row34Temp1, row34Temp2); // A3 C3 B3 D3 - __m512 row4 = _mm512_unpackhi_epi32(row34Temp1, row34Temp2); // A4 C4 B4 D4 - - row1 = _mm512_add_ps(row1, row2); - row1 = _mm512_add_ps(row1, row3); - row1 = _mm512_add_ps(row1, row4); - - return row1; - #endif - } - } - - constexpr static VectorF32<1, Packing*2> DotNoShuffle( - VectorF32 A0, VectorF32 A1, - VectorF32 C0, VectorF32 C1 - ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { - if constexpr(std::is_same_v::VectorType, __m128>) { - __m128 mulA = _mm_mul_ps(A0.v, A1.v); - __m128 mulC = _mm_mul_ps(C0.v, C1.v); - __m128i row12Temp1 = _mm_unpacklo_epi32(_mm_castps_si128(mulA), _mm_castps_si128(mulC)); // A1 C1 A2 C2 - __m128i row56Temp1 = _mm_unpackhi_epi32(_mm_castps_si128(mulA), _mm_castps_si128(mulC)); // B1 D1 B2 D2 - __m128i row1TempTemp1 = row12Temp1; - __m128i row5TempTemp1 = row56Temp1; - - row12Temp1 = _mm_unpacklo_epi32(row12Temp1, row56Temp1); // A1 B1 C1 D1 - row56Temp1 = _mm_unpackhi_epi32(row1TempTemp1, row56Temp1); // A2 B2 C2 D2 - - return _mm_add_ps(row12Temp1, row56Temp1); - } else if constexpr(std::is_same_v::VectorType, __m256>) { - __m256 mulA = _mm256_mul_ps(A0.v, A1.v); - __m256 mulC = _mm256_mul_ps(C0.v, C1.v); - __m256i row12Temp1 = _mm256_unpacklo_epi32(_mm256_castps_si256(mulA), _mm256_castps_si256(mulC)); // A1 C1 A2 C2 - __m256i row56Temp1 = _mm256_unpackhi_epi32(_mm256_castps_si256(mulA), _mm256_castps_si256(mulC)); // B1 D1 B2 D2 - __m256i row1TempTemp1 = row12Temp1; - __m256i row5TempTemp1 = row56Temp1; - - row12Temp1 = _mm256_unpacklo_epi32(row12Temp1, row56Temp1); // A1 B1 C1 D1 - row56Temp1 = _mm256_unpackhi_epi32(row1TempTemp1, row56Temp1); // A2 B2 C2 D2 - - return _mm256_add_ps(row12Temp1, row56Temp1); - #ifdef __AVX512F__ - } else { - __m512 mulA = _mm512_mul_ps(A0.v, A1.v); - __m512 mulC = _mm512_mul_ps(C0.v, C1.v); - __m512i row12Temp1 = _mm512_unpacklo_epi32(_mm512_castps_si512(mulA), _mm512_castps_si512(mulC)); // A1 C1 A2 C2 - __m512i row56Temp1 = _mm512_unpackhi_epi32(_mm512_castps_si512(mulA), _mm512_castps_si512(mulC)); // B1 D1 B2 D2 - __m512i row1TempTemp1 = row12Temp1; - __m512i row5TempTemp1 = row56Temp1; - - row12Temp1 = _mm512_unpacklo_epi32(row12Temp1, row56Temp1); // A1 B1 C1 D1 - row56Temp1 = _mm512_unpackhi_epi32(row1TempTemp1, row56Temp1); // A2 B2 C2 D2 - - return _mm512_add_ps(row12Temp1, row56Temp1); - #endif - } - } - public: - - template ShuffleValues> - constexpr static VectorF32 Blend(VectorF32 a, VectorF32 b) { - constexpr auto mask = VectorBase::template GetBlendMaskEpi32(); - - if constexpr (std::is_same_v::VectorType, __m128>) { - return _mm_castsi128_ps(_mm_blend_epi32(_mm_castps_si128(a.v), _mm_castps_si128(b.v), mask)); - - } else if constexpr (std::is_same_v::VectorType, __m256>) { - return _mm256_castsi256_ps(_mm256_blend_epi32(_mm256_castps_si256(a.v), _mm256_castps_si256(b.v), mask)); - - #ifdef __AVX512F__ - } else if constexpr (std::is_same_v::VectorType, __m512>) { - return _mm512_castsi512_ps(_mm512_mask_blend_epi32(mask, _mm512_castps_si512(a.v), _mm512_castps_si512(b.v))); - #endif - - } - } - - constexpr static VectorF32 Rotate(VectorF32<3, Packing> v, VectorF32<4, Packing> q) requires(Len == 3) { - VectorF32<3, Packing> qv(q); - VectorF32 t = Cross(qv, v) * float(2); - return v + t * q.template Shuffle<{{3,3,3,3}}>() + Cross(qv, t); - } - - constexpr static VectorF32<4, 2> RotatePivot(VectorF32<3, Packing> v, VectorF32<4, Packing> q, VectorF32<3, Packing> pivot) requires(Len == 3) { - VectorF32 translated = v - pivot; - VectorF32<3, Packing> qv(q.v); - VectorF32 t = Cross(qv, translated) * float(2); - VectorF32 rotated = translated + t * q.template Shuffle<{{3,3,3,3}}>() + Cross(qv, t); - return rotated + pivot; - } - - constexpr static VectorF32<4, Packing> QuanternionFromEuler(VectorF32<3, Packing> EulerHalf) requires(Len == 4) { - std::tuple, VectorF32<3, Packing>> sinCos = EulerHalf.SinCos(); - VectorF32<4, Packing> sin = std::get<0>(sinCos); - VectorF32<4, Packing> cos = std::get<1>(sinCos); - - VectorF32<4, Packing> row1 = cos.template Shuffle<{{0,0,0,0}}>(); - row1 = Blend<{{0,1,1,1}}>(sin, row1); - - VectorF32<4, Packing> row2 = cos.template Shuffle<{{1,1,1,1}}>(); - row2 = Blend<{{1,0,1,1}}>(sin, row2); - - row1 *= row2; - - VectorF32<4, Packing> row3 = cos.template Shuffle<{{2,2,2,2}}>(); - row3 = Blend<{{1,1,0,1}}>(sin, row3); - - row1 *= row3; - - VectorF32<4, Packing> row4 = sin.template Shuffle<{{0,0,0,0}}>(); - row4 = Blend<{{0,1,1,1}}>(cos, row4); - - VectorF32<4, Packing> row5 = sin.template Shuffle<{{1,1,1,1}}>(); - row5 = Blend<{{1,0,1,1}}>(cos, row5); - - row4 *= row5; - - VectorF32<4, Packing> row6 = sin.template Shuffle<{{2,2,2,2}}>(); - row6 = Blend<{{1,1,0,1}}>(cos, row6); - row6 = row6.template Negate<{{true,false,true,false}}>(); - - row1 = MulitplyAdd(row4, row6, row1); - - return row1; - } - }; -#elif defined(__wasm_simd128__) - // WebAssembly SIMD128 implementation. VectorType is always v128_t and we - // cap Len*Packing*sizeof(float) at 16 bytes (i.e. up to 4 floats per - // vector) in Common.cppm so a single v128_t covers every instantiation. - // Operations without a direct SIMD equivalent (Shuffle with runtime indices, - // transcendentals, etc.) round-trip through a float[4] scratch buffer. - export template - struct VectorF32 : public VectorBase { - template - friend struct VectorF32; - using Base = VectorBase; - static constexpr std::uint8_t NElems = Base::AlignmentElement; - static_assert(NElems == 4, "WASM SIMD VectorF32 assumes 4-lane vectors"); - - constexpr VectorF32() = default; - constexpr VectorF32(v128_t vv) { this->v = vv; } - constexpr VectorF32(const float* vB) { Load(vB); } - constexpr VectorF32(float val) { this->v = wasm_f32x4_splat(val); } - - constexpr void Load(const float* vB) { this->v = wasm_v128_load(vB); } - constexpr void Store(float* vB) const { wasm_v128_store(vB, this->v); } - - template - constexpr std::array Store() const { - std::array r{}; - Store(r.data()); - return r; - } - - template - constexpr operator VectorF32() const { - alignas(16) float tmp[4]; - wasm_v128_store(tmp, this->v); - alignas(16) float out[4] = {0,0,0,0}; - const std::uint8_t copyLen = (BLen < Len) ? BLen : Len; - const std::uint8_t copyPack = (BPacking < Packing) ? BPacking : Packing; - for (std::uint8_t p = 0; p < copyPack; ++p) - for (std::uint8_t i = 0; i < copyLen; ++i) - out[p * BLen + i] = tmp[p * Len + i]; - return VectorF32(wasm_v128_load(out)); - } - - constexpr VectorF32 operator+(VectorF32 b) const { return VectorF32(wasm_f32x4_add(this->v, b.v)); } - constexpr VectorF32 operator-(VectorF32 b) const { return VectorF32(wasm_f32x4_sub(this->v, b.v)); } - constexpr VectorF32 operator*(VectorF32 b) const { return VectorF32(wasm_f32x4_mul(this->v, b.v)); } - constexpr VectorF32 operator/(VectorF32 b) const { return VectorF32(wasm_f32x4_div(this->v, b.v)); } - constexpr void operator+=(VectorF32 b) { this->v = wasm_f32x4_add(this->v, b.v); } - constexpr void operator-=(VectorF32 b) { this->v = wasm_f32x4_sub(this->v, b.v); } - constexpr void operator*=(VectorF32 b) { this->v = wasm_f32x4_mul(this->v, b.v); } - constexpr void operator/=(VectorF32 b) { this->v = wasm_f32x4_div(this->v, b.v); } - - constexpr VectorF32 operator+(float b) const { return *this + VectorF32(b); } - constexpr VectorF32 operator-(float b) const { return *this - VectorF32(b); } - constexpr VectorF32 operator*(float b) const { return *this * VectorF32(b); } - constexpr VectorF32 operator/(float b) const { return *this / VectorF32(b); } - constexpr void operator+=(float b) { *this += VectorF32(b); } - constexpr void operator-=(float b) { *this -= VectorF32(b); } - constexpr void operator*=(float b) { *this *= VectorF32(b); } - constexpr void operator/=(float b) { *this /= VectorF32(b); } - - constexpr VectorF32 operator-() const { return VectorF32(wasm_f32x4_neg(this->v)); } - - constexpr bool operator==(VectorF32 b) const { - return wasm_i32x4_bitmask(wasm_f32x4_eq(this->v, b.v)) == 0b1111; - } - constexpr bool operator!=(VectorF32 b) const { return !(*this == b); } - - template - constexpr VectorF32 ExtractLo() const { - alignas(16) float tmp[4]; wasm_v128_store(tmp, this->v); - alignas(16) float out[4] = {0,0,0,0}; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < ExtractLen; ++i) - out[p * ExtractLen + i] = tmp[p * Len + i]; - return VectorF32(wasm_v128_load(out)); - } - - constexpr VectorF32 Cos() const { - alignas(16) float tmp[4]; wasm_v128_store(tmp, this->v); - for (int i = 0; i < 4; ++i) tmp[i] = std::cos(tmp[i]); - return VectorF32(wasm_v128_load(tmp)); - } - constexpr VectorF32 Sin() const { - alignas(16) float tmp[4]; wasm_v128_store(tmp, this->v); - for (int i = 0; i < 4; ++i) tmp[i] = std::sin(tmp[i]); - return VectorF32(wasm_v128_load(tmp)); - } - constexpr std::tuple, VectorF32> SinCos() const { - return { Sin(), Cos() }; - } - - template values> - constexpr VectorF32 Negate() const { - constexpr auto mask = []() { - std::array m{}; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - m[p * Len + i] = values[i] ? 0x80000000u : 0u; - return m; - }(); - v128_t maskVec = wasm_v128_load(mask.data()); - return VectorF32(wasm_v128_xor(this->v, maskVec)); - } - - static constexpr VectorF32 MulitplyAdd(VectorF32 a, VectorF32 b, VectorF32 add) { - #ifdef __wasm_relaxed_simd__ - // Single-rounded FMA (a*b + c). Host-defined when FMA hardware is - // missing — accuracy may differ from the strict-SIMD wasm path. - return VectorF32(wasm_f32x4_relaxed_madd(a.v, b.v, add.v)); - #else - return VectorF32(wasm_f32x4_add(wasm_f32x4_mul(a.v, b.v), add.v)); - #endif - } - static constexpr VectorF32 MulitplySub(VectorF32 a, VectorF32 b, VectorF32 sub) { - #ifdef __wasm_relaxed_simd__ - // a*b - c is fused as madd(a, b, -c) — same op count as mul+sub - // but one rounding instead of two. - return VectorF32(wasm_f32x4_relaxed_madd(a.v, b.v, wasm_f32x4_neg(sub.v))); - #else - return VectorF32(wasm_f32x4_sub(wasm_f32x4_mul(a.v, b.v), sub.v)); - #endif - } - - constexpr static VectorF32 Cross(VectorF32 a, VectorF32 b) requires(Len == 3) { - v128_t a_yzx = wasm_i32x4_shuffle(a.v, a.v, 1, 2, 0, 3); - v128_t a_zxy = wasm_i32x4_shuffle(a.v, a.v, 2, 0, 1, 3); - v128_t b_yzx = wasm_i32x4_shuffle(b.v, b.v, 1, 2, 0, 3); - v128_t b_zxy = wasm_i32x4_shuffle(b.v, b.v, 2, 0, 1, 3); - #ifdef __wasm_relaxed_simd__ - // a_yzx*b_zxy - a_zxy*b_yzx fused as nmadd(a_zxy, b_yzx, a_yzx*b_zxy) - // = -(a_zxy*b_yzx) + a_yzx*b_zxy. Replaces a mul+sub pair with a - // single FMA. - return VectorF32(wasm_f32x4_relaxed_nmadd(a_zxy, b_yzx, wasm_f32x4_mul(a_yzx, b_zxy))); - #else - return VectorF32(wasm_f32x4_sub(wasm_f32x4_mul(a_yzx, b_zxy), wasm_f32x4_mul(a_zxy, b_yzx))); - #endif - } - - template ShuffleValues> - constexpr VectorF32 Shuffle() const { - alignas(16) float tmp[4]; wasm_v128_store(tmp, this->v); - alignas(16) float out[4] = {0,0,0,0}; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - out[p * Len + i] = tmp[p * Len + ShuffleValues[i]]; - return VectorF32(wasm_v128_load(out)); - } - - template ShuffleValues> - constexpr static VectorF32 Blend(VectorF32 a, VectorF32 b) { - constexpr auto mask = []() { - std::array m{}; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - m[p * Len + i] = ShuffleValues[i] ? 0xFFFFFFFFu : 0u; - return m; - }(); - v128_t maskVec = wasm_v128_load(mask.data()); - return VectorF32(wasm_v128_bitselect(b.v, a.v, maskVec)); - } - - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto LengthSq(VectorF32 first, Rest... rest) { - constexpr std::uint8_t N = VectorBase::BatchSize; - VectorF32<1, static_cast(Packing * N)> r; - std::array, N> args{ first, rest... }; - alignas(16) float buf[4] = {0,0,0,0}; - for (std::uint8_t i = 0; i < N; ++i) { - alignas(16) float tmp[4]; - wasm_v128_store(tmp, args[i].v); - for (std::uint8_t p = 0; p < Packing; ++p) { - float acc = 0.0f; - for (std::uint8_t k = 0; k < Len; ++k) { - float x = tmp[p * Len + k]; - acc += x * x; - } - buf[i * Packing + p] = acc; - } - } - r.v = wasm_v128_load(buf); - return r; - } - - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto Length(VectorF32 first, Rest... rest) { - auto sq = LengthSq(first, rest...); - sq.v = wasm_f32x4_sqrt(sq.v); - return sq; - } - - // Pairwise dot products packed into one v128. Only the first Len - // lanes contribute, so the same routine handles 3- and 4-component - // inputs — the 4th lane of Len==3 inputs may be garbage from Cross() - // and must not be summed. Takes BatchSize pairs (== 4 here since - // WASM AlignmentElement is always 4 and Packing must be 1). - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == 2 * VectorBase::BatchSize) && - (Len == 3 || Len == 4) && Packing == 1) - constexpr static VectorF32<1, 4> Dot(VectorF32 first, Rest... rest) { - constexpr std::uint8_t N = VectorBase::BatchSize; - std::array, 2 * N> args{ first, rest... }; - alignas(16) float out[4] = {0,0,0,0}; - for (std::uint8_t i = 0; i < N; ++i) { - alignas(16) float a[4], b[4]; - wasm_v128_store(a, args[2 * i].v); - wasm_v128_store(b, args[2 * i + 1].v); - for (std::uint8_t k = 0; k < Len; ++k) out[i] += a[k] * b[k]; - } - return VectorF32<1, 4>(wasm_v128_load(out)); - } - - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto Normalize(VectorF32 first, Rest... rest) { - auto normOne = [](VectorF32 u) { - alignas(16) float tmp[4]; wasm_v128_store(tmp, u.v); - alignas(16) float out[4] = {0,0,0,0}; - for (std::uint8_t p = 0; p < Packing; ++p) { - float acc = 0.0f; - for (std::uint8_t k = 0; k < Len; ++k) { - float x = tmp[p * Len + k]; - acc += x * x; - } - float invLen = acc > 0.0f ? 1.0f / std::sqrt(acc) : 0.0f; - for (std::uint8_t k = 0; k < Len; ++k) - out[p * Len + k] = tmp[p * Len + k] * invLen; - } - return VectorF32(wasm_v128_load(out)); - }; - return std::array, VectorBase::BatchSize>{ normOne(first), normOne(rest)... }; - } - - constexpr static VectorF32 Rotate(VectorF32<3, Packing> v, VectorF32<4, Packing> q) requires(Len == 3) { - alignas(16) float qBuf[4]; wasm_v128_store(qBuf, q.v); - alignas(16) float qvBuf[4] = {0,0,0,0}; - alignas(16) float qwBuf[4] = {0,0,0,0}; - for (std::uint8_t p = 0; p < Packing; ++p) { - qvBuf[p * 3 + 0] = qBuf[p * 4 + 0]; - qvBuf[p * 3 + 1] = qBuf[p * 4 + 1]; - qvBuf[p * 3 + 2] = qBuf[p * 4 + 2]; - for (std::uint8_t i = 0; i < 3; ++i) qwBuf[p * 3 + i] = qBuf[p * 4 + 3]; - } - VectorF32<3, Packing> qv(wasm_v128_load(qvBuf)); - VectorF32<3, Packing> qwBroadcast(wasm_v128_load(qwBuf)); - VectorF32<3, Packing> t = Cross(qv, v) * 2.0f; - return v + t * qwBroadcast + Cross(qv, t); - } - - constexpr static VectorF32<3, Packing> RotatePivot(VectorF32<3, Packing> v, VectorF32<4, Packing> q, VectorF32<3, Packing> pivot) requires(Len == 3) { - VectorF32<3, Packing> translated = v - pivot; - return Rotate(translated, q) + pivot; - } - - constexpr static VectorF32<4, Packing> QuanternionFromEuler(VectorF32<3, Packing> eulerHalf) requires(Len == 4) { - alignas(16) float eulerBuf[4]; wasm_v128_store(eulerBuf, eulerHalf.v); - alignas(16) float outBuf[4] = {0,0,0,0}; - for (std::uint8_t p = 0; p < Packing; ++p) { - float roll = eulerBuf[p * 3 + 0]; - float pitch = eulerBuf[p * 3 + 1]; - float yaw = eulerBuf[p * 3 + 2]; - float sr = std::sin(roll), cr = std::cos(roll); - float sp = std::sin(pitch), cp = std::cos(pitch); - float sy = std::sin(yaw), cy = std::cos(yaw); - outBuf[p * 4 + 0] = sr * cp * cy - cr * sp * sy; - outBuf[p * 4 + 1] = cr * sp * cy + sr * cp * sy; - outBuf[p * 4 + 2] = cr * cp * sy - sr * sp * cy; - outBuf[p * 4 + 3] = cr * cp * cy + sr * sp * sy; - } - return VectorF32<4, Packing>(wasm_v128_load(outBuf)); - } - }; -#elif defined(__riscv_vector) - // RISC-V V extension implementation. Storage is a GNU vector of 16/32/64 - // bytes (picked in Common.cppm from the guaranteed VLEN); native operators - // map to vfadd/vfsub/vfmul/vfdiv. Per-element loops compile to vrgather/ - // vmerge/vfsqrt when the autovectoriser can see the pattern, and to - // scalar fallback otherwise. Hand-tuned intrinsic paths - // (e.g. vsetvl + vfwmacc for batched dot) can land incrementally. - export template - struct VectorF32 : public VectorBase { - template - friend struct VectorF32; - using Base = VectorBase; - static constexpr std::uint8_t NElems = Base::AlignmentElement; - - constexpr VectorF32() = default; - constexpr VectorF32(typename Base::VectorType vv) { this->v = vv; } - constexpr VectorF32(const float* vB) { Load(vB); } - constexpr VectorF32(float val) { - for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = val; - } - - constexpr void Load(const float* vB) { - for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = vB[i]; - } - constexpr void Store(float* vB) const { - for (std::uint8_t i = 0; i < NElems; ++i) vB[i] = this->v[i]; - } - - template - constexpr std::array Store() const { - std::array r{}; - Store(r.data()); - return r; - } - - template - constexpr operator VectorF32() const { - VectorF32 r; - const std::uint8_t copyLen = (BLen < Len) ? BLen : Len; - const std::uint8_t copyPack = (BPacking < Packing) ? BPacking : Packing; - for (std::uint8_t p = 0; p < copyPack; ++p) - for (std::uint8_t i = 0; i < copyLen; ++i) - r.v[p * BLen + i] = this->v[p * Len + i]; - return r; - } - - constexpr VectorF32 operator+(VectorF32 b) const { return VectorF32(this->v + b.v); } - constexpr VectorF32 operator-(VectorF32 b) const { return VectorF32(this->v - b.v); } - constexpr VectorF32 operator*(VectorF32 b) const { return VectorF32(this->v * b.v); } - constexpr VectorF32 operator/(VectorF32 b) const { return VectorF32(this->v / b.v); } - constexpr void operator+=(VectorF32 b) { this->v = this->v + b.v; } - constexpr void operator-=(VectorF32 b) { this->v = this->v - b.v; } - constexpr void operator*=(VectorF32 b) { this->v = this->v * b.v; } - constexpr void operator/=(VectorF32 b) { this->v = this->v / b.v; } - - constexpr VectorF32 operator+(float b) const { return *this + VectorF32(b); } - constexpr VectorF32 operator-(float b) const { return *this - VectorF32(b); } - constexpr VectorF32 operator*(float b) const { return *this * VectorF32(b); } - constexpr VectorF32 operator/(float b) const { return *this / VectorF32(b); } - constexpr void operator+=(float b) { *this += VectorF32(b); } - constexpr void operator-=(float b) { *this -= VectorF32(b); } - constexpr void operator*=(float b) { *this *= VectorF32(b); } - constexpr void operator/=(float b) { *this /= VectorF32(b); } - - constexpr VectorF32 operator-() const { return VectorF32(-this->v); } - - constexpr bool operator==(VectorF32 b) const { - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - if (this->v[p * Len + i] != b.v[p * Len + i]) return false; - return true; - } - constexpr bool operator!=(VectorF32 b) const { return !(*this == b); } - - template - constexpr VectorF32 ExtractLo() const { - VectorF32 r; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < ExtractLen; ++i) - r.v[p * ExtractLen + i] = this->v[p * Len + i]; - return r; - } - - constexpr VectorF32 Cos() const { - VectorF32 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = std::cos(this->v[i]); - return r; - } - constexpr VectorF32 Sin() const { - VectorF32 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = std::sin(this->v[i]); - return r; - } - constexpr std::tuple, VectorF32> SinCos() const { - return { Sin(), Cos() }; - } - - template values> - constexpr VectorF32 Negate() const { - VectorF32 r; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - r.v[p * Len + i] = values[i] ? -this->v[p * Len + i] : this->v[p * Len + i]; - return r; - } - - // a*b + c — the compiler fuses to vfmacc.vv under the default - // -ffp-contract=on. No explicit intrinsic needed. - static constexpr VectorF32 MulitplyAdd(VectorF32 a, VectorF32 b, VectorF32 add) { - return VectorF32(a.v * b.v + add.v); - } - static constexpr VectorF32 MulitplySub(VectorF32 a, VectorF32 b, VectorF32 sub) { - return VectorF32(a.v * b.v - sub.v); - } - - constexpr static VectorF32 Cross(VectorF32 a, VectorF32 b) requires(Len == 3) { - VectorF32 r; - for (std::uint8_t p = 0; p < Packing; ++p) { - const std::uint8_t base = p * 3; - r.v[base + 0] = a.v[base + 1] * b.v[base + 2] - a.v[base + 2] * b.v[base + 1]; - r.v[base + 1] = a.v[base + 2] * b.v[base + 0] - a.v[base + 0] * b.v[base + 2]; - r.v[base + 2] = a.v[base + 0] * b.v[base + 1] - a.v[base + 1] * b.v[base + 0]; - } - return r; - } - - template ShuffleValues> - constexpr VectorF32 Shuffle() const { - VectorF32 r; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - r.v[p * Len + i] = this->v[p * Len + ShuffleValues[i]]; - return r; - } - - template ShuffleValues> - constexpr static VectorF32 Blend(VectorF32 a, VectorF32 b) { - VectorF32 r; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - r.v[p * Len + i] = ShuffleValues[i] ? b.v[p * Len + i] : a.v[p * Len + i]; - return r; - } - - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto LengthSq(VectorF32 first, Rest... rest) { - constexpr std::uint8_t N = VectorBase::BatchSize; - VectorF32<1, static_cast(Packing * N)> r; - std::array, N> args{ first, rest... }; - for (std::uint8_t i = 0; i < N; ++i) - for (std::uint8_t p = 0; p < Packing; ++p) { - float acc = 0.0f; - for (std::uint8_t k = 0; k < Len; ++k) { - float x = args[i].v[p * Len + k]; - acc += x * x; - } - r.v[i * Packing + p] = acc; - } - return r; - } - - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto Length(VectorF32 first, Rest... rest) { - auto sq = LengthSq(first, rest...); - for (std::uint8_t i = 0; i < decltype(sq)::NElems; ++i) sq.v[i] = std::sqrt(sq.v[i]); - return sq; - } - - // Pairwise dot products across BatchSize pairs. The 4th lane of Len==3 - // inputs may carry garbage from Cross(), so only the first Len lanes - // are summed per pair. - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == 2 * VectorBase::BatchSize)) - constexpr static auto Dot(VectorF32 first, Rest... rest) { - constexpr std::uint8_t N = VectorBase::BatchSize; - VectorF32<1, static_cast(Packing * N)> r; - std::array, 2 * N> args{ first, rest... }; - for (std::uint8_t i = 0; i < N; ++i) - for (std::uint8_t p = 0; p < Packing; ++p) { - float acc = 0.0f; - for (std::uint8_t k = 0; k < Len; ++k) - acc += args[2 * i].v[p * Len + k] * args[2 * i + 1].v[p * Len + k]; - r.v[i * Packing + p] = acc; - } - return r; - } - - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto Normalize(VectorF32 first, Rest... rest) { - auto normOne = [](VectorF32 u) { - VectorF32 out; - for (std::uint8_t p = 0; p < Packing; ++p) { - float acc = 0.0f; - for (std::uint8_t k = 0; k < Len; ++k) { - float x = u.v[p * Len + k]; - acc += x * x; - } - float invLen = acc > 0.0f ? 1.0f / std::sqrt(acc) : 0.0f; - for (std::uint8_t k = 0; k < Len; ++k) - out.v[p * Len + k] = u.v[p * Len + k] * invLen; - } - return out; - }; - return std::array, VectorBase::BatchSize>{ normOne(first), normOne(rest)... }; - } - - constexpr static VectorF32 Rotate(VectorF32<3, Packing> v, VectorF32<4, Packing> q) requires(Len == 3) { - VectorF32<3, Packing> qv; - VectorF32<3, Packing> qwBroadcast; - for (std::uint8_t p = 0; p < Packing; ++p) { - qv.v[p * 3 + 0] = q.v[p * 4 + 0]; - qv.v[p * 3 + 1] = q.v[p * 4 + 1]; - qv.v[p * 3 + 2] = q.v[p * 4 + 2]; - for (std::uint8_t i = 0; i < 3; ++i) qwBroadcast.v[p * 3 + i] = q.v[p * 4 + 3]; - } - VectorF32<3, Packing> t = Cross(qv, v) * 2.0f; - return v + t * qwBroadcast + Cross(qv, t); - } - - constexpr static VectorF32<3, Packing> RotatePivot(VectorF32<3, Packing> v, VectorF32<4, Packing> q, VectorF32<3, Packing> pivot) requires(Len == 3) { - VectorF32<3, Packing> translated = v - pivot; - return Rotate(translated, q) + pivot; - } - - constexpr static VectorF32<4, Packing> QuanternionFromEuler(VectorF32<3, Packing> eulerHalf) requires(Len == 4) { - VectorF32<4, Packing> r; - for (std::uint8_t p = 0; p < Packing; ++p) { - float roll = eulerHalf.v[p * 3 + 0]; - float pitch = eulerHalf.v[p * 3 + 1]; - float yaw = eulerHalf.v[p * 3 + 2]; - float sr = std::sin(roll), cr = std::cos(roll); - float sp = std::sin(pitch), cp = std::cos(pitch); - float sy = std::sin(yaw), cy = std::cos(yaw); - r.v[p * 4 + 0] = sr * cp * cy - cr * sp * sy; - r.v[p * 4 + 1] = cr * sp * cy + sr * cp * sy; - r.v[p * 4 + 2] = cr * cp * sy - sr * sp * cy; - r.v[p * 4 + 3] = cr * cp * cy + sr * sp * sy; - } - return r; - } - }; -#else - // Scalar software fallback for non-x86_64 targets. Future arches can swap - // in their own intrinsic implementation by adding an arch-specific branch - // above and gating this one out. - export template - struct VectorF32 : public VectorBase { - template - friend struct VectorF32; - using Base = VectorBase; - static constexpr std::uint8_t NElems = Base::AlignmentElement; - - constexpr VectorF32() = default; - constexpr VectorF32(typename Base::VectorType vv) { - this->v = vv; - } - constexpr VectorF32(const float* vB) { Load(vB); } -#ifdef __FLT16_MAX__ - constexpr VectorF32(const _Float16* vB) { Load(vB); } -#endif - constexpr VectorF32(float val) { - for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = val; - } - - constexpr void Load(const float* vB) { - for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = vB[i]; - } - constexpr void Store(float* vB) const { - for (std::uint8_t i = 0; i < NElems; ++i) vB[i] = this->v[i]; - } -#ifdef __FLT16_MAX__ - constexpr void Load(const _Float16* vB) { - for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = static_cast(vB[i]); - } - constexpr void Store(_Float16* vB) const { - for (std::uint8_t i = 0; i < NElems; ++i) vB[i] = static_cast<_Float16>(this->v[i]); - } -#endif - - template - constexpr std::array Store() const { - std::array r{}; - Store(r.data()); - return r; - } - - template - constexpr operator VectorF32() const { - VectorF32 r; - const std::uint8_t copyLen = (BLen < Len) ? BLen : Len; - const std::uint8_t copyPack = (BPacking < Packing) ? BPacking : Packing; - for (std::uint8_t p = 0; p < copyPack; ++p) - for (std::uint8_t i = 0; i < copyLen; ++i) - r.v[p * BLen + i] = this->v[p * Len + i]; - return r; - } - - constexpr VectorF32 operator+(VectorF32 b) const { - VectorF32 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] + b.v[i]; - return r; - } - constexpr VectorF32 operator-(VectorF32 b) const { - VectorF32 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] - b.v[i]; - return r; - } - constexpr VectorF32 operator*(VectorF32 b) const { - VectorF32 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] * b.v[i]; - return r; - } - constexpr VectorF32 operator/(VectorF32 b) const { - VectorF32 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] / b.v[i]; - return r; - } - - constexpr void operator+=(VectorF32 b) { for (std::uint8_t i=0;iv[i] += b.v[i]; } - constexpr void operator-=(VectorF32 b) { for (std::uint8_t i=0;iv[i] -= b.v[i]; } - constexpr void operator*=(VectorF32 b) { for (std::uint8_t i=0;iv[i] *= b.v[i]; } - constexpr void operator/=(VectorF32 b) { for (std::uint8_t i=0;iv[i] /= b.v[i]; } - - constexpr VectorF32 operator+(float b) const { return *this + VectorF32(b); } - constexpr VectorF32 operator-(float b) const { return *this - VectorF32(b); } - constexpr VectorF32 operator*(float b) const { return *this * VectorF32(b); } - constexpr VectorF32 operator/(float b) const { return *this / VectorF32(b); } - constexpr void operator+=(float b) { *this += VectorF32(b); } - constexpr void operator-=(float b) { *this -= VectorF32(b); } - constexpr void operator*=(float b) { *this *= VectorF32(b); } - constexpr void operator/=(float b) { *this /= VectorF32(b); } - - constexpr VectorF32 operator-() const { - VectorF32 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = -this->v[i]; - return r; - } - - constexpr bool operator==(VectorF32 b) const { - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - if (this->v[p * Len + i] != b.v[p * Len + i]) return false; - return true; - } - constexpr bool operator!=(VectorF32 b) const { return !(*this == b); } - - template - constexpr VectorF32 ExtractLo() const { - VectorF32 r; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < ExtractLen; ++i) - r.v[p * ExtractLen + i] = this->v[p * Len + i]; - return r; - } - - constexpr VectorF32 Cos() const { - VectorF32 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = std::cos(this->v[i]); - return r; - } - constexpr VectorF32 Sin() const { - VectorF32 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = std::sin(this->v[i]); - return r; - } - constexpr std::tuple, VectorF32> SinCos() const { - return { Sin(), Cos() }; - } - - template values> - constexpr VectorF32 Negate() const { - VectorF32 r; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - r.v[p * Len + i] = values[i] ? -this->v[p * Len + i] : this->v[p * Len + i]; - return r; - } - - static constexpr VectorF32 MulitplyAdd(VectorF32 a, VectorF32 b, VectorF32 add) { - VectorF32 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = a.v[i] * b.v[i] + add.v[i]; - return r; - } - static constexpr VectorF32 MulitplySub(VectorF32 a, VectorF32 b, VectorF32 sub) { - VectorF32 r; - for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = a.v[i] * b.v[i] - sub.v[i]; - return r; - } - - constexpr static VectorF32 Cross(VectorF32 a, VectorF32 b) requires(Len == 3) { - VectorF32 r; - for (std::uint8_t p = 0; p < Packing; ++p) { - const std::uint8_t base = p * 3; - r.v[base + 0] = a.v[base + 1] * b.v[base + 2] - a.v[base + 2] * b.v[base + 1]; - r.v[base + 1] = a.v[base + 2] * b.v[base + 0] - a.v[base + 0] * b.v[base + 2]; - r.v[base + 2] = a.v[base + 0] * b.v[base + 1] - a.v[base + 1] * b.v[base + 0]; - } - return r; - } - - template ShuffleValues> - constexpr VectorF32 Shuffle() const { - VectorF32 r; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - r.v[p * Len + i] = this->v[p * Len + ShuffleValues[i]]; - return r; - } - - template ShuffleValues> - constexpr static VectorF32 Blend(VectorF32 a, VectorF32 b) { - VectorF32 r; - for (std::uint8_t p = 0; p < Packing; ++p) - for (std::uint8_t i = 0; i < Len; ++i) - r.v[p * Len + i] = ShuffleValues[i] ? b.v[p * Len + i] : a.v[p * Len + i]; - return r; - } - - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto LengthSq(VectorF32 first, Rest... rest) { - constexpr std::uint8_t N = VectorBase::BatchSize; - VectorF32<1, static_cast(Packing * N)> r; - std::array, N> args{ first, rest... }; - for (std::uint8_t i = 0; i < N; ++i) - for (std::uint8_t p = 0; p < Packing; ++p) { - float acc = 0.0f; - for (std::uint8_t k = 0; k < Len; ++k) { - float x = args[i].v[p * Len + k]; - acc += x * x; - } - r.v[i * Packing + p] = acc; - } - return r; - } - - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto Length(VectorF32 first, Rest... rest) { - auto sq = LengthSq(first, rest...); - for (std::uint8_t i = 0; i < decltype(sq)::NElems; ++i) sq.v[i] = std::sqrt(sq.v[i]); - return sq; - } - - template - requires((std::is_same_v> && ...) && - (1 + sizeof...(Rest) == VectorBase::BatchSize)) - constexpr static auto Normalize(VectorF32 first, Rest... rest) { - auto normOne = [](VectorF32 u) { - VectorF32 out; - for (std::uint8_t p = 0; p < Packing; ++p) { - float acc = 0.0f; - for (std::uint8_t k = 0; k < Len; ++k) { - float x = u.v[p * Len + k]; - acc += x * x; - } - float invLen = acc > 0.0f ? 1.0f / std::sqrt(acc) : 0.0f; - for (std::uint8_t k = 0; k < Len; ++k) - out.v[p * Len + k] = u.v[p * Len + k] * invLen; - } - return out; - }; - return std::array, VectorBase::BatchSize>{ normOne(first), normOne(rest)... }; - } - - constexpr static VectorF32 Rotate(VectorF32<3, Packing> v, VectorF32<4, Packing> q) requires(Len == 3) { - VectorF32<3, Packing> qv; - VectorF32<3, Packing> qwBroadcast; - for (std::uint8_t p = 0; p < Packing; ++p) { - qv.v[p * 3 + 0] = q.v[p * 4 + 0]; - qv.v[p * 3 + 1] = q.v[p * 4 + 1]; - qv.v[p * 3 + 2] = q.v[p * 4 + 2]; - for (std::uint8_t i = 0; i < 3; ++i) qwBroadcast.v[p * 3 + i] = q.v[p * 4 + 3]; - } - VectorF32<3, Packing> t = Cross(qv, v) * 2.0f; - return v + t * qwBroadcast + Cross(qv, t); - } - - constexpr static VectorF32<3, Packing> RotatePivot(VectorF32<3, Packing> v, VectorF32<4, Packing> q, VectorF32<3, Packing> pivot) requires(Len == 3) { - VectorF32<3, Packing> translated = v - pivot; - return Rotate(translated, q) + pivot; - } - - constexpr static VectorF32<4, Packing> QuanternionFromEuler(VectorF32<3, Packing> eulerHalf) requires(Len == 4) { - VectorF32<4, Packing> r; - for (std::uint8_t p = 0; p < Packing; ++p) { - float roll = eulerHalf.v[p * 3 + 0]; - float pitch = eulerHalf.v[p * 3 + 1]; - float yaw = eulerHalf.v[p * 3 + 2]; - float sr = std::sin(roll), cr = std::cos(roll); - float sp = std::sin(pitch), cp = std::cos(pitch); - float sy = std::sin(yaw), cy = std::cos(yaw); - r.v[p * 4 + 0] = sr * cp * cy - cr * sp * sy; - r.v[p * 4 + 1] = cr * sp * cy + sr * cp * sy; - r.v[p * 4 + 2] = cr * cp * sy - sr * sp * cy; - r.v[p * 4 + 3] = cr * cp * cy + sr * sp * sy; - } - return r; - } - }; -#endif -} - -export template -struct std::formatter> : std::formatter { - constexpr auto format(const Crafter::VectorF32& obj, format_context& ctx) const { - std::array::AlignmentElement> vec = obj.template Store(); - std::string out = "{"; - for(std::uint32_t i = 0; i < Packing; i++) { - out += "{"; - for(std::uint32_t i2 = 0; i2 < Len; i2++) { - out += std::format("{}", static_cast(vec[i * Len + i2])); - if (i2 + 1 < Len) out += ","; - } - out += "}"; - } - out += "}"; - return std::formatter::format(out, ctx); - } +//SPDX-License-Identifier: LGPL-3.0-only +//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® + +module; +#ifdef __x86_64 +#include +#endif +#ifdef __wasm_simd128__ +#include +#endif +#ifdef __riscv_vector +#include +#endif +export module Crafter.Math:VectorF32; +import std; +import :Common; + +namespace Crafter { +#ifdef __x86_64 + export template + struct VectorF32 : public VectorBase { + template + friend struct VectorF32; + + constexpr VectorF32() = default; + constexpr VectorF32(VectorBase::VectorType v) { + this->v = v; + } + constexpr VectorF32(const float* vB) { + Load(vB); + }; + constexpr VectorF32(const _Float16* vB) { + Load(vB); + }; + constexpr VectorF32(float val) { + if constexpr(std::is_same_v::VectorType, __m128>) { + this->v = _mm_set1_ps(val); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + this->v = _mm256_set1_ps(val); + } else { + this->v = _mm512_set1_ps(val); + } + }; + constexpr void Load(const float* vB) { + if constexpr(std::is_same_v::VectorType, __m128>) { + this->v = _mm_loadu_ps(vB); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + this->v = _mm256_loadu_ps(vB); + } else { + this->v = _mm512_loadu_ps(vB); + } + } + constexpr void Store(float* vB) const { + if constexpr(std::is_same_v::VectorType, __m128>) { + _mm_storeu_ps(vB, this->v); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + _mm256_storeu_ps(vB, this->v); + } else { + _mm512_storeu_ps(vB, this->v); + } + } + + constexpr void Load(const _Float16* vB) { + #ifdef __F16C__ + if constexpr (std::is_same_v::VectorType, __m128>) { + this->v = _mm_cvtph_ps(_mm_loadl_epi64(reinterpret_cast(vB))); + } else if constexpr (std::is_same_v::VectorType, __m256>) { + this->v = _mm256_cvtph_ps(_mm_loadu_si128(reinterpret_cast(vB))); + } else { + this->v = _mm512_cvtph_ps(_mm256_loadu_si256(reinterpret_cast(vB))); + } + #else + alignas(64) float tmp[Len]; + for (int i = 0; i < Len; ++i) + tmp[i] = static_cast(vB[i]); + if constexpr (std::is_same_v::VectorType, __m128>) { + this->v = _mm_load_ps(tmp); + } else if constexpr (std::is_same_v::VectorType, __m256>) { + this->v = _mm256_load_ps(tmp); + } else { + this->v = _mm512_load_ps(tmp); + } + #endif + } + + constexpr void Store(_Float16* vB) const { + #ifdef __F16C__ + if constexpr (std::is_same_v::VectorType, __m128>) { + _mm_storel_epi64(reinterpret_cast<__m128i*>(vB), _mm_cvtps_ph(this->v, _MM_FROUND_TO_NEAREST_INT)); + } else if constexpr (std::is_same_v::VectorType, __m256>) { + _mm_storeu_si128(reinterpret_cast<__m128i*>(vB), _mm256_cvtps_ph(this->v, _MM_FROUND_TO_NEAREST_INT)); + } else { + _mm256_storeu_si256(reinterpret_cast<__m256i*>(vB), _mm512_cvtps_ph(this->v, _MM_FROUND_TO_NEAREST_INT)); + } + #else + alignas(64) float tmp[Len]; + if constexpr (std::is_same_v::VectorType, __m128>) { + _mm_store_ps(tmp, this->v); + } else if constexpr (std::is_same_v::VectorType, __m256>) { + _mm256_store_ps(tmp, this->v); + } else { + _mm512_store_ps(tmp, this->v); + } + for (int i = 0; i < Len; ++i) + vB[i] = static_cast<_Float16>(tmp[i]); + #endif + } + + template + constexpr std::array::AlignmentElement> Store() const { + std::array::AlignmentElement> returnArray; + Store(returnArray.data()); + return returnArray; + } + + template + constexpr operator VectorF32() const { + if constexpr (Len == BLen) { + if constexpr(std::is_same_v::VectorType, __m256> && std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm256_castps256_ps128(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m512> && std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm512_castps512_ps128(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m512> && std::is_same_v::VectorType, __m256>) { + return VectorF32(_mm512_castps512_ps256(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m128> && std::is_same_v::VectorType, __m256>) { + return VectorF32(_mm256_castps128_ps256(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m128> && std::is_same_v::VectorType, __m512>) { + return VectorF32(_mm512_castps128_ps512(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m256> && std::is_same_v::VectorType, __m512>) { + return VectorF32(_mm512_castps256_ps512(this->v)); + } else { + return VectorF32(this->v); + } + } else if constexpr (BLen <= Len) { + return this->template ExtractLo(); + } else { + if constexpr(std::is_same_v::VectorType, __m128>) { + if constexpr(std::is_same_v::VectorType, __m128>) { + constexpr std::array::Alignment> shuffleMask = VectorBase::template GetExtractLoMaskEpi8(); + __m128i shuffleVec = _mm_loadu_si128(reinterpret_cast(shuffleMask.data())); + return VectorF32(_mm_castsi128_ps(_mm_shuffle_epi8(_mm_castps_si128(this->v), shuffleVec))); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + constexpr std::array::AlignmentElement> permMask =VectorBase::template GetExtractLoMaskepi32(); + __m256i permIdx = _mm256_loadu_si256(reinterpret_cast(permMask.data())); + __m256i result = _mm256_permutexvar_epi32(permIdx, _mm_castps_si256(this->v)); + return VectorF32(_mm_castsi128_ps(_mm256_castsi256_si128(result))); + #ifdef __AVX512F__ + } else { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); + __m512i permIdx = _mm512_loadu_epi32(permMask.data()); + __m512i result = _mm512_permutexvar_epi32(permIdx, _mm512_castps_si512(this->v)); + return VectorF32(_mm_castsi128_ps(_mm512_castsi512_si128(result))); + #endif + } + } else if constexpr(std::is_same_v::VectorType, __m256>) { + if constexpr(std::is_same_v::VectorType, __m128>) { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); + __m256i permIdx = _mm256_loadu_si256(reinterpret_cast(permMask.data())); + __m256i result = _mm256_permutexvar_epi32(permIdx, _mm256_castsi128_si256(_mm_castps_si128(this->v))); + return VectorF32(_mm256_castsi256_ps(result)); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); + __m256i permIdx = _mm256_loadu_si256(reinterpret_cast(permMask.data())); + __m256i result = _mm256_permutexvar_epi32(permIdx, _mm256_castps_si256(this->v)); + return VectorF32(_mm256_castsi256_ps(result)); + #ifdef __AVX512F__ + } else { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); + __m256i permIdx = _mm512_loadu_epi32(permMask.data()); + __m256i result = _mm512_permutexvar_epi32(permIdx, _mm512_castsi512_si256(_mm512_castps_si512(this->v))); + return VectorF32(_mm256_castsi256_ps(result)); + #endif + } + #ifdef __AVX512F__ + } else { + if constexpr(std::is_same_v::VectorType, __m128>) { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); + __m512i permIdx = _mm512_loadu_epi32(permMask.data()); + __m512i result = _mm512_permutexvar_epi32(permIdx, _mm512_castsi128_si512(_mm_castps_si128(this->v))); + return VectorF32(_mm512_castsi512_ps(result)); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); + __m512i permIdx = _mm512_loadu_epi32(permMask.data()); + __m512i result = _mm512_permutexvar_epi32(permIdx, _mm512_castsi256_si512(_mm256_castps_si256(this->v))); + return VectorF32(_mm512_castsi512_ps(result)); + } else { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); + __m512i permIdx = _mm512_loadu_epi32(permMask.data()); + __m512i result = _mm512_permutexvar_epi32(permIdx, _mm512_castps_si512(this->v)); + return VectorF32(_mm512_castsi512_ps(result)); + } + #endif + } + } + } + + constexpr VectorF32 operator+(VectorF32 b) const { + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm_add_ps(this->v, b.v)); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32(_mm256_add_ps(this->v, b.v)); + } else { + return VectorF32(_mm512_add_ps(this->v, b.v)); + } + } + + constexpr VectorF32 operator-(VectorF32 b) const { + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm_sub_ps(this->v, b.v)); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32(_mm256_sub_ps(this->v, b.v)); + } else { + return VectorF32(_mm512_sub_ps(this->v, b.v)); + } + } + + constexpr VectorF32 operator*(VectorF32 b) const { + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm_mul_ps(this->v, b.v)); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32(_mm256_mul_ps(this->v, b.v)); + } else { + return VectorF32(_mm512_mul_ps(this->v, b.v)); + } + } + + constexpr VectorF32 operator/(VectorF32 b) const { + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm_div_ps(this->v, b.v)); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32(_mm256_div_ps(this->v, b.v)); + } else { + return VectorF32(_mm512_div_ps(this->v, b.v)); + } + } + + + constexpr void operator+=(VectorF32 b) { + if constexpr(std::is_same_v::VectorType, __m128>) { + this->v = _mm_add_ps(this->v, b.v); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + this->v = _mm256_add_ps(this->v, b.v); + } else { + this->v = _mm512_add_ps(this->v, b.v); + } + } + + constexpr void operator-=(VectorF32 b) { + if constexpr(std::is_same_v::VectorType, __m128>) { + this->v = _mm_sub_ps(this->v, b.v); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + this->v = _mm256_sub_ps(this->v, b.v); + } else { + this->v = _mm512_sub_ps(this->v, b.v); + } + } + + constexpr void operator*=(VectorF32 b) { + if constexpr(std::is_same_v::VectorType, __m128>) { + this->v = _mm_mul_ps(this->v, b.v); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + this->v = _mm256_mul_ps(this->v, b.v); + } else { + this->v = _mm512_mul_ps(this->v, b.v); + } + } + + constexpr void operator/=(VectorF32 b) { + if constexpr(std::is_same_v::VectorType, __m128>) { + this->v = _mm_div_ps(this->v, b.v); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + this->v = _mm256_div_ps(this->v, b.v); + } else { + this->v = _mm512_div_ps(this->v, b.v); + } + } + + constexpr VectorF32 operator+(float b) { + VectorF32 vB(b); + return *this + vB; + } + + constexpr VectorF32 operator-(float b) { + VectorF32 vB(b); + return *this - vB; + } + + constexpr VectorF32 operator*(float b) { + VectorF32 vB(b); + return *this * vB; + } + + constexpr VectorF32 operator/(float b) { + VectorF32 vB(b); + return *this / vB; + } + + constexpr void operator+=(float b) { + VectorF32 vB(b); + *this += vB; + } + + constexpr void operator-=(float b) { + VectorF32 vB(b); + *this -= vB; + } + + constexpr void operator*=(float b) { + VectorF32 vB(b); + *this *= vB; + } + + constexpr void operator/=(float b) { + VectorF32 vB(b); + *this /= vB; + } + + constexpr VectorF32 operator-(){ + return Negate::GetAllTrue()>(); + } + + constexpr bool operator==(VectorF32 b) const { + if constexpr(std::is_same_v::VectorType, __m128>) { + #ifdef __AVX512VL__ + return _mm_cmp_ps_mask(this->v, b.v, _CMP_EQ_OQ) == 0xF; + #else + return _mm_movemask_ps(_mm_cmpeq_ps(this->v, b.v)) == 0xF; + #endif + } else if constexpr(std::is_same_v::VectorType, __m256>) { + #ifdef __AVX512VL__ + return _mm256_cmp_ps_mask(this->v, b.v, _CMP_EQ_OQ) == 0xFF; + #else + return _mm256_movemask_ps(_mm256_cmp_ps(this->v, b.v, _CMP_EQ_OQ)) == 0xFF; + #endif + } else { + return _mm512_cmp_ps_mask(this->v, b.v, _CMP_EQ_OQ) == 0xFFFF; + } + } + + constexpr bool operator!=(VectorF32 b) const { + return !(*this == b); + } + + template + constexpr VectorF32 ExtractLo() const { + if constexpr(Packing > 1) { + if constexpr(std::is_same_v::VectorType, __m128>) { + constexpr std::array::Alignment> shuffleMask = VectorBase::template GetExtractLoMaskEpi8(); + __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); + return VectorF32(_mm_castsi128_ps(_mm_shuffle_epi8(_mm_castps_si128(this->v), shuffleVec))); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); + __m256i permIdx = _mm256_loadu_si256(reinterpret_cast(permMask.data())); + __m256i result = _mm256_permutexvar_epi32(permIdx, _mm256_castps_si256(this->v)); + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm256_castps256_ps128(_mm256_castsi256_ps(result))); + } else { + return VectorF32(_mm256_castsi256_ps(result)); + } + } else { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetExtractLoMaskEpi32(); + __m512i permIdx = _mm512_loadu_epi32(permMask.data()); + __m512i result = _mm512_permutexvar_epi32(permIdx, _mm512_castps_si512(this->v)); + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm512_castps512_ps128(_mm512_castsi512_ps(result))); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32(_mm512_castps512_ps256(_mm512_castsi512_ps(result))); + } else { + return VectorF32(_mm512_castsi512_ps(result)); + } + } + } else { + if constexpr(std::is_same_v::VectorType, __m256> && std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm256_castps256_ps128(this->v)); + #ifdef __AVX512F__ + } else if constexpr(std::is_same_v::VectorType, __m512> && std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm512_castps512_ps128(this->v)); + } else if constexpr(std::is_same_v::VectorType, __m512> && std::is_same_v::VectorType, __m256>) { + return VectorF32(_mm512_castps512_ps256(this->v)); + #endif + } else { + return VectorF32(this->v); + } + } + } + + constexpr VectorF32 Cos() { + if constexpr (std::is_same_v::VectorType, __m128>) { + return VectorF32(VectorBase::cos_f32x4(this->v)); + } else if constexpr (std::is_same_v::VectorType, __m256>) { + return VectorF32(VectorBase::cos_f32x8(this->v)); + #ifdef __AVX512F__ + } else { + return VectorF32(VectorBase::cos_f32x16(this->v)); + #endif + } + } + + constexpr VectorF32 Sin() { + if constexpr (std::is_same_v::VectorType, __m128>) { + return VectorF32(VectorBase::sin_f32x4(this->v)); + } else if constexpr (std::is_same_v::VectorType, __m256>) { + return VectorF32(VectorBase::sin_f32x8(this->v)); + #ifdef __AVX512F__ + } else { + return VectorF32(VectorBase::sin_f32x16(this->v)); + #endif + } + } + + std::tuple, VectorF32> SinCos() { + if constexpr (std::is_same_v::VectorType, __m128>) { + __m128 s, c; + VectorBase::sincos_f32x4(this->v, s, c); + return { + VectorF32(s), + VectorF32(c) + }; + + } else if constexpr (std::is_same_v::VectorType, __m256>) { + __m256 s, c; + VectorBase::sincos_f32x8(this->v, s, c); + return { + VectorF32(s), + VectorF32(c) + }; + #ifdef __AVX512F__ + } else { + __m512 s, c; + VectorBase::sincos_f32x16(this->v, s, c); + return { + VectorF32(s), + VectorF32(c) + }; + #endif + } + } + + template values> + constexpr VectorF32 Negate() const { + std::array::AlignmentElement> mask = VectorBase::template GetNegateMask(); + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm_castsi128_ps(_mm_xor_si128(_mm_castps_si128(this->v), _mm_loadu_si128(reinterpret_cast<__m128i*>(mask.data()))))); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32(_mm256_castsi256_ps(_mm256_xor_si256(_mm256_castps_si256(this->v), _mm256_loadu_si256(reinterpret_cast<__m256i*>(mask.data()))))); + #ifdef __AVX512F__ + } else { + return VectorF32(_mm512_castsi512_ps(_mm512_xor_si512(_mm512_castps_si512(this->v), _mm512_loadu_epi32(mask.data())))); + #endif + } + } + + static constexpr VectorF32 MulitplyAdd(VectorF32 a, VectorF32 b, VectorF32 add) { + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm_fmadd_ps(a.v, b.v, add.v)); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32(_mm256_fmadd_ps(a.v, b.v, add.v)); + #ifdef __AVX512F__ + } else { + return VectorF32(_mm512_fmadd_ps(a.v, b.v, add.v)); + #endif + } + } + + static constexpr VectorF32 MulitplySub(VectorF32 a, VectorF32 b, VectorF32 sub) { + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm_fmsub_ps(a.v, b.v, sub.v)); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32(_mm256_fmsub_ps(a.v, b.v, sub.v)); + #ifdef __AVX512F__ + } else { + return VectorF32(_mm512_fmsub_ps(a.v, b.v, sub.v)); + #endif + } + } + + constexpr static VectorF32 Cross(VectorF32 a, VectorF32 b) requires(Len == 3) { + VectorF32 row1 = a.template Shuffle<{{1,2,0}}>(); + VectorF32 row4 = b.template Shuffle<{{1,2,0}}>(); + + VectorF32 row3 = a.template Shuffle<{{2,0,1}}>(); + VectorF32 row2 = b.template Shuffle<{{2,0,1}}>(); + + VectorF32 result = row3 * row4; + return VectorF32::MulitplySub(row1, row2, result); + } + + template ShuffleValues> + constexpr VectorF32 Shuffle() { + if constexpr(VectorBase::template CheckEpi32Shuffle()) { + constexpr std::uint8_t imm = VectorBase::template GetShuffleMaskEpi32(); + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32(_mm_castsi128_ps(_mm_shuffle_epi32(_mm_castps_si128(this->v), imm))); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32(_mm256_castsi256_ps(_mm256_shuffle_epi32(_mm256_castps_si256(this->v), imm))); + #ifdef __AVX512F__ + } else { + return VectorF32(_mm512_castsi512_ps(_mm512_shuffle_epi32(_mm512_castps_si512(this->v), imm))); + #endif + } + } else if constexpr(VectorBase::template CheckEpi8Shuffle()) { + constexpr std::array::Alignment> shuffleMask = VectorBase::template GetShuffleMaskEpi8(); + if constexpr(std::is_same_v::VectorType, __m128>) { + __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); + return VectorF32(_mm_castsi128_ps(_mm_shuffle_epi8(_mm_castps_si128(this->v), shuffleVec))); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + #ifdef __AVX512BW__ + __m256i shuffleVec = _mm256_loadu_si256(reinterpret_cast(shuffleMask.data())); + return VectorF32(_mm256_castsi256_ps( _mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castps_si256(this->v)),_mm512_castsi256_si512(shuffleVec))))); + #else + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetPermuteMaskEpi32(); + __m256i permIdx = _mm256_loadu_si256(reinterpret_cast(permMask.data())); + return VectorF32(_mm256_castsi256_ps(_mm256_permutevar8x32_epi32(_mm256_castps_si256(this->v), permIdx))); + #endif + #ifdef __AVX512F__ + } else { + __m512i shuffleVec = _mm512_loadu_si512(reinterpret_cast(shuffleMask.data())); + return VectorF32(_mm512_castsi512_ps(_mm512_shuffle_epi8(_mm512_castps_si512(this->v), shuffleVec))); + #endif + } + } else { + if constexpr(std::is_same_v::VectorType, __m128>) { + constexpr std::array::Alignment> shuffleMask = VectorBase::template GetShuffleMaskEpi8(); + __m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data()); + return VectorF32(_mm_castsi128_ps(_mm_shuffle_epi8(_mm_castps_si128(this->v), shuffleVec))); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetPermuteMaskEpi32(); + __m256i permIdx = _mm256_loadu_si256(reinterpret_cast(permMask.data())); + return VectorF32(_mm256_castsi256_ps(_mm256_permutevar8x32_epi32(_mm256_castps_si256(this->v), permIdx))); + #ifdef __AVX512F__ + } else { + constexpr std::array::AlignmentElement> permMask = VectorBase::template GetPermuteMaskEpi32(); + __m512i permIdx = _mm512_loadu_epi32(permMask.data()); + return VectorF32(_mm512_castsi512_ps(_mm512_permutexvar_epi32(permIdx, _mm512_castps_si512(this->v)))); + #endif + } + } + } + + // Public variadic surface — one name per op, arity locked to BatchSize. + // The Pack helpers below carry the SIMD bodies and the per-(Len,Packing) + // requires clauses; this wrapper just forwards once arity matches. + template + requires ((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto Normalize(VectorF32 first, Rest... rest) { + return NormalizePack(first, rest...); + } + + template + requires ((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto Length(VectorF32 first, Rest... rest) { + return LengthPack(first, rest...); + } + + template + requires ((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto LengthSq(VectorF32 first, Rest... rest) { + return LengthSqPack(first, rest...); + } + + template + requires ((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == 2 * VectorBase::BatchSize)) + constexpr static auto Dot(VectorF32 first, Rest... rest) { + return DotPack(first, rest...); + } + + private: + constexpr static std::array, VectorBase::BatchSize> NormalizePack( + VectorF32 A, + VectorF32 B, + VectorF32 C, + VectorF32 D + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + if constexpr(std::is_same_v::VectorType, __m128>) { + VectorF32<1, 4> lenght = LengthNoShuffle(A, C, B, D); + constexpr float oneArr[] {1, 1, 1, 1}; + __m128 one = _mm_loadu_ps(oneArr); + VectorF32<4, 1> fLenght(_mm_div_ps(one, lenght.v)); + + VectorF32<4, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0}}>(); + VectorF32<4, 1> fLenghtB = fLenght.template Shuffle<{{1,1,1,1}}>(); + VectorF32<4, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2}}>(); + VectorF32<4, 1> fLenghtD = fLenght.template Shuffle<{{3,3,3,3}}>(); + + return { + _mm_mul_ps(A.v, fLenghtA.v), + _mm_mul_ps(B.v, fLenghtB.v), + _mm_mul_ps(C.v, fLenghtC.v), + _mm_mul_ps(D.v, fLenghtD.v) + }; + } else if constexpr(std::is_same_v::VectorType, __m256>) { + VectorF32<1, 8> lenght = LengthNoShuffle(A, C, B, D); + constexpr float oneArr[] {1, 1, 1, 1, 1, 1, 1, 1}; + __m256 one = _mm256_loadu_ps(oneArr); + VectorF32<8, 1> fLenght(_mm256_div_ps(one, lenght.v)); + + + VectorF32<8, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,4,4,4,4}}>(); + VectorF32<8, 1> fLenghtB = fLenght.template Shuffle<{{1,1,1,1,5,5,5,5}}>(); + VectorF32<8, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,6,6,6,6}}>(); + VectorF32<8, 1> fLenghtD = fLenght.template Shuffle<{{3,3,3,3,7,7,7,7}}>(); + + return { + _mm256_mul_ps(A.v, fLenghtA.v), + _mm256_mul_ps(B.v, fLenghtB.v), + _mm256_mul_ps(C.v, fLenghtC.v), + _mm256_mul_ps(D.v, fLenghtD.v) + }; + #if defined(__AVX512F__) + } else { + VectorF32<1, 16> lenght = LengthNoShuffle(A, C, B, D); + constexpr float oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + __m512 one = _mm512_loadu_ps(oneArr); + VectorF32<16, 1> fLenght(_mm512_div_ps(one, lenght.v)); + VectorF32<16, 1> fLenght2(lenght.v); + + VectorF32<16, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,4,4,4,4,8,8,8,8,12,12,12,12}}>(); + VectorF32<16, 1> fLenghtB = fLenght.template Shuffle<{{1,1,1,1,5,5,5,5,9,9,9,9,13,13,13,13}}>(); + VectorF32<16, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,6,6,6,6,10,10,10,10,14,14,14,14}}>(); + VectorF32<16, 1> fLenghtD = fLenght.template Shuffle<{{3,3,3,3,7,7,7,7,11,11,11,11,15,15,15,15}}>(); + + + return { + VectorF32(_mm512_mul_ps(A.v, fLenghtA.v)), + VectorF32(_mm512_mul_ps(B.v, fLenghtB.v)), + VectorF32(_mm512_mul_ps(C.v, fLenghtC.v)), + VectorF32(_mm512_mul_ps(D.v, fLenghtD.v)), + }; + #endif + } + } + + constexpr static std::array, VectorBase::BatchSize> NormalizePack( + VectorF32 A, + VectorF32 B, + VectorF32 C, + VectorF32 D + ) requires(Len == 3 && Packing == 1) { + VectorF32<1, 4> lenght = Length(A, B, C, D); + constexpr float oneArr[] {1, 1, 1, 1}; + __m128 one = _mm_loadu_ps(oneArr); + VectorF32<4, 1> fLenght(_mm_div_ps(one, lenght.v)); + + VectorF32<4, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0}}>(); + VectorF32<4, 1> fLenghtB = fLenght.template Shuffle<{{1,1,1,1}}>(); + VectorF32<4, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2}}>(); + VectorF32<4, 1> fLenghtD = fLenght.template Shuffle<{{3,3,3,3}}>(); + + return { + _mm_mul_ps(A.v, fLenghtA.v), + _mm_mul_ps(B.v, fLenghtB.v), + _mm_mul_ps(C.v, fLenghtC.v), + _mm_mul_ps(D.v, fLenghtD.v) + }; + } + + constexpr static std::array, VectorBase::BatchSize> NormalizePack( + VectorF32 A, + VectorF32 B, + VectorF32 C, + VectorF32 D + ) requires(Len == 3 && Packing == 2) { + VectorF32<1, 8> lenght = Length(A, B, C, D); + constexpr float oneArr[] {1, 1, 1, 1, 1, 1, 1, 1}; + __m256 one = _mm256_loadu_ps(oneArr); + VectorF32<8, 1> fLenght(_mm256_div_ps(one, lenght.v)); + + VectorF32<8, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0, 1,1,1}}>(); + VectorF32<8, 1> fLenghtB = fLenght.template Shuffle<{{2,2,2, 3,3,3}}>(); + VectorF32<8, 1> fLenghtC = fLenght.template Shuffle<{{4,4,4, 5,5,5}}>(); + VectorF32<8, 1> fLenghtD = fLenght.template Shuffle<{{6,6,6, 7,7,7}}>(); + + return { + _mm256_mul_ps(A.v, fLenghtA.v), + _mm256_mul_ps(B.v, fLenghtB.v), + _mm256_mul_ps(C.v, fLenghtC.v), + _mm256_mul_ps(D.v, fLenghtD.v) + }; + } + + #ifdef __AVX512F__ + constexpr static std::array, VectorBase::BatchSize> NormalizePack( + VectorF32 A, + VectorF32 B, + VectorF32 C + ) requires(Len == 3 && Packing == 5) { + VectorF32<1, 15> lenght = Length(A, B, C); + constexpr float oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + __m512 one = _mm512_loadu_ps(oneArr); + VectorF32<15, 1> fLenght(_mm512_div_ps(one, lenght.v)); + + VectorF32<15, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0, 1,1,1, 2,2,2, 3,3,3, 4,4,4}}>(); + VectorF32<15, 1> fLenghtB = fLenght.template Shuffle<{{5,5,5, 6,6,6, 7,7,7, 8,8,8, 9,9,9}}>(); + VectorF32<15, 1> fLenghtC = fLenght.template Shuffle<{{10,10,10, 11,11,11, 12,12,12, 13,13,13, 14,14,14}}>(); + + return { + _mm512_mul_ps(A.v, fLenghtA.v), + _mm512_mul_ps(B.v, fLenghtB.v), + _mm512_mul_ps(C.v, fLenghtC.v), + }; + } + #endif + + constexpr static std::array, VectorBase::BatchSize> NormalizePack( + VectorF32 A, + VectorF32 B + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + if constexpr(std::is_same_v::VectorType, __m128>) { + VectorF32<1, 4> lenght = LengthNoShuffle(A, B); + constexpr float oneArr[] {1, 1, 1, 1}; + __m128 one = _mm_loadu_ps(oneArr); + VectorF32<4, 1> fLenght(_mm_div_ps(one, lenght.v)); + + VectorF32<4, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1}}>(); + VectorF32<4, 1> fLenghtB = fLenght.template Shuffle<{{2,2,3,3}}>(); + + return { + _mm_mul_ps(A.v, fLenghtA.v), + _mm_mul_ps(B.v, fLenghtB.v), + }; + } else if constexpr(std::is_same_v::VectorType, __m256>) { + VectorF32<1, 8> lenght = LengthNoShuffle(A, B); + constexpr float oneArr[] {1, 1, 1, 1, 1, 1, 1, 1}; + __m256 one = _mm256_loadu_ps(oneArr); + VectorF32<8, 1> fLenght(_mm256_div_ps(one, lenght.v)); + + VectorF32<8, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,4,4,5,5}}>(); + VectorF32<8, 1> fLenghtB = fLenght.template Shuffle<{{2,2,3,3,6,6,7,7}}>(); + + return { + _mm256_mul_ps(A.v, fLenghtA.v), + _mm256_mul_ps(B.v, fLenghtB.v), + }; + #ifdef __AVX512F__ + } else { + VectorF32<1, 16> lenght = LengthNoShuffle(A, B); + constexpr float oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}; + __m512 one = _mm512_loadu_ps(oneArr); + VectorF32<16, 1> fLenght(_mm512_div_ps(one, lenght.v)); + + VectorF32<16, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,4,4,5,5,8,8,9,9,12,12,13,13}}>(); + VectorF32<16, 1> fLenghtB = fLenght.template Shuffle<{{2,2,3,3,6,6,7,7,10,10,11,11,14,14,15,15}}>(); + + return { + _mm512_mul_ps(A.v, fLenghtA.v), + _mm512_mul_ps(B.v, fLenghtB.v), + }; + #endif + } + } + + constexpr static VectorF32<1, Packing*4> LengthPack( + VectorF32 A, + VectorF32 B, + VectorF32 C, + VectorF32 D + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + VectorF32<1, Packing*4> lenghtSq = LengthSqPack(A, B, C, D); + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32<1, Packing*4>(_mm_sqrt_ps(lenghtSq.v)); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32<1, Packing*4>(_mm256_sqrt_ps(lenghtSq.v)); + } else { + return VectorF32<1, Packing*4>(_mm512_sqrt_ps(lenghtSq.v)); + } + } + + constexpr static VectorF32<1, 4> LengthPack( + VectorF32 A, + VectorF32 B, + VectorF32 C, + VectorF32 D + ) requires(Len == 3 && Packing == 1) { + VectorF32<1, 4> lenghtSq = LengthSqPack(A, B, C, D); + return VectorF32<1, 4>(_mm_sqrt_ps(lenghtSq.v)); + } + + constexpr static VectorF32<1, 8> LengthPack( + VectorF32 A, + VectorF32 B, + VectorF32 C, + VectorF32 D + ) requires(Len == 3 && Packing == 2) { + VectorF32<1, 8> lenghtSq = LengthSqPack(A, B, C, D); + return VectorF32<1, Packing*4>(_mm256_sqrt_ps(lenghtSq.v)); + } + + #ifdef __AVX512F__ + constexpr static VectorF32<1, 15> LengthPack( + VectorF32 A, + VectorF32 B, + VectorF32 C + ) requires(Len == 3 && Packing == 5) { + VectorF32<1, 15> lenghtSq = LengthSqPack(A, B, C); + return VectorF32<1, 15>(_mm512_sqrt_ps(lenghtSq.v)); + } + #endif + + constexpr static VectorF32<1, Packing*2> LengthPack( + VectorF32 A, + VectorF32 C + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + VectorF32<1, Packing*2> lenghtSq = LengthSqPack(A, C); + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32<1, Packing*2>(_mm_sqrt_ps(lenghtSq.v)); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32<1, Packing*2>(_mm256_sqrt_ps(lenghtSq.v)); + #ifdef __AVX512F__ + } else { + return VectorF32<1, Packing*2>(_mm512_sqrt_ps(lenghtSq.v)); + #endif + } + } + + constexpr static VectorF32<1, Packing*4> LengthSqPack( + VectorF32 A, + VectorF32 B, + VectorF32 C, + VectorF32 D + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + return DotPack(A, A, B, B, C, C, D, D); + } + + constexpr static VectorF32<1, 4> LengthSqPack( + VectorF32 A, + VectorF32 B, + VectorF32 C, + VectorF32 D + ) requires(Len == 3 && Packing == 1) { + return DotPack(A, A, B, B, C, C, D, D); + } + + constexpr static VectorF32<1, 8> LengthSqPack( + VectorF32 A, + VectorF32 B, + VectorF32 C, + VectorF32 D + ) requires(Len == 3 && Packing == 2) { + return DotPack(A, A, B, B, C, C, D, D); + } + + #ifdef __AVX512F__ + constexpr static VectorF32<1, 15> LengthSqPack( + VectorF32 A, + VectorF32 B, + VectorF32 C + ) requires(Len == 3 && Packing == 5) { + return DotPack(A, A, B, B, C, C); + } + #endif + + constexpr static VectorF32<1, Packing*2> LengthSqPack( + VectorF32 A, + VectorF32 C + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + return DotPack(A, A, C, C); + } + + constexpr static VectorF32<1, Packing*4> DotPack( + VectorF32 A0, VectorF32 A1, + VectorF32 B0, VectorF32 B1, + VectorF32 C0, VectorF32 C1, + VectorF32 D0, VectorF32 D1 + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + if constexpr(std::is_same_v::VectorType, __m128>) { + return DotNoShuffle(A0, A1, C0, C1, B0, B1, D0, D1); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + VectorF32<8, 1> vec(DotNoShuffle(A0, A1, B0, B1, C0, C1, D0, D1).v); + vec = vec.template Shuffle<{{ + 0,4,2,6, + 1,5,3,7, + }}>(); + return vec.v; + #ifdef __AVX512F__ + } else { + VectorF32<16, 1> vec(DotNoShuffle(A0, A1, B0, B1, C0, C1, D0, D1).v); + vec = vec.template Shuffle<{{ + 0,4,8,12, + 2,6,10,14, + 1,5,9,13, + 3,7,11,15 + }}>(); + return vec.v; + #endif + } + } + + constexpr static VectorF32<1, 4> DotPack( + VectorF32 A0, VectorF32 A1, + VectorF32 B0, VectorF32 B1, + VectorF32 C0, VectorF32 C1, + VectorF32 D0, VectorF32 D1 + ) requires(Len == 3 && Packing == 1) { + // Each register: [X1 X2 X3 _] + // 4 pairs (A,B,C,D) → 4 dot products → 1 x __m128 + // + // After element-wise multiply: + // mulA = [a1 a2 a3 _] (where ai = A0[i]*A1[i]) + // mulB = [b1 b2 b3 _] + // mulC = [c1 c2 c3 _] + // mulD = [d1 d2 d3 _] + // + // We need: result = [a1+a2+a3, b1+b2+b3, c1+c2+c3, d1+d2+d3] + // + // Transpose to get: + // row1 = [a1 b1 c1 d1] + // row2 = [a2 b2 c2 d2] + // row3 = [a3 b3 c3 d3] + // Then sum rows. + + __m128 mulA = _mm_mul_ps(A0.v, A1.v); + __m128 mulB = _mm_mul_ps(B0.v, B1.v); + __m128 mulC = _mm_mul_ps(C0.v, C1.v); + __m128 mulD = _mm_mul_ps(D0.v, D1.v); + + // Standard 4x4 transpose (only first 3 rows matter, 4th is garbage) + // unpacklo/hi interleave pairs of 32-bit elements + __m128 tmp0 = _mm_unpacklo_ps(mulA, mulB); // a1 b1 a2 b2 + __m128 tmp1 = _mm_unpackhi_ps(mulA, mulB); // a3 b3 _ _ + __m128 tmp2 = _mm_unpacklo_ps(mulC, mulD); // c1 d1 c2 d2 + __m128 tmp3 = _mm_unpackhi_ps(mulC, mulD); // c3 d3 _ _ + + __m128 row1 = _mm_movelh_ps(tmp0, tmp2); // a1 b1 c1 d1 + __m128 row2 = _mm_movehl_ps(tmp2, tmp0); // a2 b2 c2 d2 + __m128 row3 = _mm_movelh_ps(tmp1, tmp3); // a3 b3 c3 d3 + + row1 = _mm_add_ps(row1, row2); + row1 = _mm_add_ps(row1, row3); + + return row1; + } + + constexpr static VectorF32<1, 8> DotPack( + VectorF32 A0, VectorF32 A1, + VectorF32 B0, VectorF32 B1, + VectorF32 C0, VectorF32 C1, + VectorF32 D0, VectorF32 D1 + ) requires(Len == 3 && Packing == 2) { + // Each register: [X1 X2 X3 Y1 Y2 Y3 _ _] + // 4 pairs × 2 vectors each = 8 dot products → 1 x __m256 + // + // After multiply: + // mulA = [a1 a2 a3 b1 b2 b3 _ _] + // mulB = [c1 c2 c3 d1 d2 d3 _ _] + // mulC = [e1 e2 e3 f1 f2 f3 _ _] + // mulD = [g1 g2 g3 h1 h2 h3 _ _] + // + // We need result = [a·, b·, c·, d·, e·, f·, g·, h·] + // where x· = x1+x2+x3 + // + // Strategy: use permute to gather element 1s, 2s, 3s across all 8 vectors, + // then add. + // + // Gather indices (from the concatenated view of mulA|mulB|mulC|mulD): + // vec: a a a b b b _ _ c c c d d d _ _ e e e f f f _ _ g g g h h h _ _ + // idx: 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 + // + // elem1 = [a1, b1, c1, d1, e1, f1, g1, h1] → indices [0, 3, 8, 11, 16, 19, 24, 27] + // elem2 = [a2, b2, c2, d2, e2, f2, g2, h2] → indices [1, 4, 9, 12, 17, 20, 25, 28] + // elem3 = [a3, b3, c3, d3, e3, f3, g3, h3] → indices [2, 5, 10, 13, 18, 21, 26, 29] + // + // Unfortunately AVX2 doesn't have cross-register permutes for 8x32 easily. + // Use vpermd (_mm256_permutevar8x32) within pairs, then blend/combine. + // + // Within each 256-bit register [X1 X2 X3 Y1 Y2 Y3 _ _]: + // elem1_local = [X1 Y1 ...] → gather from indices 0,3 + // elem2_local = [X2 Y2 ...] → gather from indices 1,4 + // elem3_local = [X3 Y3 ...] → gather from indices 2,5 + // + // After permutevar8x32 on each mul register: + // From mulA: row1_part = [a1 b1 _ _ _ _ _ _] + // From mulB: row1_part = [c1 d1 _ _ _ _ _ _] + // From mulC: row1_part = [e1 f1 _ _ _ _ _ _] + // From mulD: row1_part = [g1 h1 _ _ _ _ _ _] + // + // Then combine with unpack/shuffle to get full rows. + + __m256 mulA = _mm256_mul_ps(A0.v, A1.v); // a1 a2 a3 b1 b2 b3 _ _ + __m256 mulB = _mm256_mul_ps(B0.v, B1.v); // c1 c2 c3 d1 d2 d3 _ _ + __m256 mulC = _mm256_mul_ps(C0.v, C1.v); // e1 e2 e3 f1 f2 f3 _ _ + __m256 mulD = _mm256_mul_ps(D0.v, D1.v); // g1 g2 g3 h1 h2 h3 _ _ + + // Permute each register to gather elements by position. + // For each register [X1 X2 X3 Y1 Y2 Y3 U U]: + // perm1: [X1 Y1 X2 Y2 X3 Y3 _ _] → indices {0,3,1,4,2,5,6,7} + __m256i permIdx = _mm256_setr_epi32(0, 3, 1, 4, 2, 5, 6, 7); + + // After permute: [X1 Y1 X2 Y2 X3 Y3 _ _] + __m256 pA = _mm256_permutevar8x32_ps(mulA, permIdx); // a1 b1 a2 b2 a3 b3 _ _ + __m256 pB = _mm256_permutevar8x32_ps(mulB, permIdx); // c1 d1 c2 d2 c3 d3 _ _ + __m256 pC = _mm256_permutevar8x32_ps(mulC, permIdx); // e1 f1 e2 f2 e3 f3 _ _ + __m256 pD = _mm256_permutevar8x32_ps(mulD, permIdx); // g1 h1 g2 h2 g3 h3 _ _ + + // Now combine pairs. Each pair contributes 4 consecutive results. + // pA has [a1 b1 a2 b2 a3 b3 _ _], pB has [c1 d1 c2 d2 c3 d3 _ _] + // We want: + // row1 = [a1 b1 c1 d1 | e1 f1 g1 h1] + // row2 = [a2 b2 c2 d2 | e2 f2 g2 h2] + // row3 = [a3 b3 c3 d3 | e3 f3 g3 h3] + // + // From pA: elements at [0,1] are elem1, [2,3] are elem2, [4,5] are elem3 + // From pB: elements at [0,1] are elem1, [2,3] are elem2, [4,5] are elem3 + // + // Use unpacklo_epi64 to interleave 64-bit chunks: + // unpacklo64(pA, pB) within 128-bit lanes: + // lo lane: pA[0:1]=a1,b1 | pB[0:1]=c1,d1 → [a1 b1 c1 d1] + // hi lane: pA[4:5]=a3,b3 | pB[4:5]=c3,d3 → [a3 b3 c3 d3] + // → [a1 b1 c1 d1 | a3 b3 c3 d3] + // + // unpackhi64(pA, pB) within 128-bit lanes: + // lo lane: pA[2:3]=a2,b2 | pB[2:3]=c2,d2 → [a2 b2 c2 d2] + // hi lane: pA[6:7]=_,_ | pB[6:7]=_,_ → garbage + // → [a2 b2 c2 d2 | _ _ _ _] + + __m256i AB_lo = _mm256_unpacklo_epi64( + _mm256_castps_si256(pA), _mm256_castps_si256(pB)); // [a1 b1 c1 d1 | a3 b3 c3 d3] + __m256i AB_hi = _mm256_unpackhi_epi64( + _mm256_castps_si256(pA), _mm256_castps_si256(pB)); // [a2 b2 c2 d2 | _ _ _ _] + + __m256i CD_lo = _mm256_unpacklo_epi64( + _mm256_castps_si256(pC), _mm256_castps_si256(pD)); // [e1 f1 g1 h1 | e3 f3 g3 h3] + __m256i CD_hi = _mm256_unpackhi_epi64( + _mm256_castps_si256(pC), _mm256_castps_si256(pD)); // [e2 f2 g2 h2 | _ _ _ _] + + // row1 = [a1 b1 c1 d1 | e1 f1 g1 h1] → lo 128 of AB_lo, lo 128 of CD_lo + // row2 = [a2 b2 c2 d2 | e2 f2 g2 h2] → lo 128 of AB_hi, lo 128 of CD_hi + // row3 = [a3 b3 c3 d3 | e3 f3 g3 h3] → hi 128 of AB_lo, hi 128 of CD_lo + + __m256 row1 = _mm256_castsi256_ps(_mm256_permute2x128_si256(AB_lo, CD_lo, 0x20)); // lo,lo + __m256 row2 = _mm256_castsi256_ps(_mm256_permute2x128_si256(AB_hi, CD_hi, 0x20)); // lo,lo + __m256 row3 = _mm256_castsi256_ps(_mm256_permute2x128_si256(AB_lo, CD_lo, 0x31)); // hi,hi + + row1 = _mm256_add_ps(row1, row2); + row1 = _mm256_add_ps(row1, row3); + + return row1; + } + + #ifdef __AVX512F__ + constexpr static VectorF32<1, 15> DotPack( + VectorF32 A0, VectorF32 A1, + VectorF32 B0, VectorF32 B1, + VectorF32 C0, VectorF32 C1 + ) requires(Len == 3 && Packing == 5) { + // __m512: Each register: [A1 A2 A3 B1 B2 B3 C1 C2 C3 D1 D2 D3 E1 E2 E3 _] + // 3 pairs × 5 vectors each = 15 dot products → fits in 1 x __m512 (slot 16 unused) + // + // After multiply of 3 pairs: + // mul0 = [a1 a2 a3 b1 b2 b3 c1 c2 c3 d1 d2 d3 e1 e2 e3 _] + // mul1 = [f1 f2 f3 g1 g2 g3 h1 h2 h3 i1 i2 i3 j1 j2 j3 _] + // mul2 = [k1 k2 k3 l1 l2 l3 m1 m2 m3 n1 n2 n3 o1 o2 o3 _] + // + // Result = [a· b· c· d· e· f· g· h· i· j· k· l· m· n· o· _] + // + // Strategy: for each mul register, gather element 1s, 2s, 3s with vpermps, + // then combine across registers. + // + // From mul0: 5 vectors at positions {0,1,2}, {3,4,5}, {6,7,8}, {9,10,11}, {12,13,14} + // elem1 = indices {0, 3, 6, 9, 12} → positions 0..4 of result + // elem2 = indices {1, 4, 7, 10, 13} + // elem3 = indices {2, 5, 8, 11, 14} + + __m512 mul0 = _mm512_mul_ps(A0.v, A1.v); + __m512 mul1 = _mm512_mul_ps(B0.v, B1.v); + __m512 mul2 = _mm512_mul_ps(C0.v, C1.v); + + // Gather elem1, elem2, elem3 from each mul register + // Each register has 5 vec3s: extract element 1,2,3 of each into consecutive positions + __m512i idx1 = _mm512_setr_epi32(0, 3, 6, 9, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + __m512i idx2 = _mm512_setr_epi32(1, 4, 7, 10, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + __m512i idx3 = _mm512_setr_epi32(2, 5, 8, 11, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0); + + // From mul0 → results 0..4, from mul1 → results 5..9, from mul2 → results 10..14 + // Gather from each, then combine. + + __m512 e1_0 = _mm512_permutexvar_ps(idx1, mul0); // [a1 b1 c1 d1 e1 ...] + __m512 e2_0 = _mm512_permutexvar_ps(idx2, mul0); // [a2 b2 c2 d2 e2 ...] + __m512 e3_0 = _mm512_permutexvar_ps(idx3, mul0); // [a3 b3 c3 d3 e3 ...] + + __m512 e1_1 = _mm512_permutexvar_ps(idx1, mul1); // [f1 g1 h1 i1 j1 ...] + __m512 e2_1 = _mm512_permutexvar_ps(idx2, mul1); // [f2 g2 h2 i2 j2 ...] + __m512 e3_1 = _mm512_permutexvar_ps(idx3, mul1); // [f3 g3 h3 i3 j3 ...] + + __m512 e1_2 = _mm512_permutexvar_ps(idx1, mul2); // [k1 l1 m1 n1 o1 ...] + __m512 e2_2 = _mm512_permutexvar_ps(idx2, mul2); // [k2 l2 m2 n2 o2 ...] + __m512 e3_2 = _mm512_permutexvar_ps(idx3, mul2); // [k3 l3 m3 n3 o3 ...] + + // Now combine: we need positions 0..4 from reg0, 5..9 from reg1, 10..14 from reg2 + // Use masked moves to assemble the final row vectors. + // mask for positions 0-4: 0b0000000000011111 = 0x001F + // mask for positions 5-9: 0b0000001111100000 = 0x03E0 + // mask for positions 10-14: 0b0111110000000000 = 0x7C00 + + // For reg1, its results are in positions 0..4 but need to go to 5..9. + // For reg2, its results are in positions 0..4 but need to go to 10..14. + // Use a different approach: permute reg1/reg2 results to their target positions. + + // Shift reg1 results from slots 0..4 to slots 5..9 + __m512i shiftIdx1 = _mm512_setr_epi32(0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 0, 0, 0, 0, 0, 0); + // Shift reg2 results from slots 0..4 to slots 10..14 + __m512i shiftIdx2 = _mm512_setr_epi32(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 3, 4, 0); + + __m512 e1_1_shifted = _mm512_permutexvar_ps(shiftIdx1, e1_1); + __m512 e2_1_shifted = _mm512_permutexvar_ps(shiftIdx1, e2_1); + __m512 e3_1_shifted = _mm512_permutexvar_ps(shiftIdx1, e3_1); + + __m512 e1_2_shifted = _mm512_permutexvar_ps(shiftIdx2, e1_2); + __m512 e2_2_shifted = _mm512_permutexvar_ps(shiftIdx2, e2_2); + __m512 e3_2_shifted = _mm512_permutexvar_ps(shiftIdx2, e3_2); + + // Blend: take positions 0..4 from reg0, 5..9 from reg1, 10..14 from reg2 + __mmask16 mask_5_9 = 0x03E0u; // bits 5-9 + __mmask16 mask_10_14 = 0x7C00u; // bits 10-14 + + __m512 row1 = _mm512_mask_mov_ps(e1_0, mask_5_9, e1_1_shifted); + row1 = _mm512_mask_mov_ps(row1, mask_10_14, e1_2_shifted); + + __m512 row2 = _mm512_mask_mov_ps(e2_0, mask_5_9, e2_1_shifted); + row2 = _mm512_mask_mov_ps(row2, mask_10_14, e2_2_shifted); + + __m512 row3 = _mm512_mask_mov_ps(e3_0, mask_5_9, e3_1_shifted); + row3 = _mm512_mask_mov_ps(row3, mask_10_14, e3_2_shifted); + + row1 = _mm512_add_ps(row1, row2); + row1 = _mm512_add_ps(row1, row3); + + return row1; + } + #endif + + constexpr static VectorF32<1, Packing*2> DotPack( + VectorF32 A0, VectorF32 A1, + VectorF32 C0, VectorF32 C1 + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + if constexpr(std::is_same_v::VectorType, __m128>) { + return DotNoShuffle(A0, A1, C0, C1); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + VectorF32<8, 1> vec(DotNoShuffle(A0, A1, C0, C1).v); + vec = vec.template Shuffle<{{ + 0,1, 4,5, + 2,3, 6,7, + }}>(); + return vec.v; + #ifdef __AVX512F__ + } else { + VectorF32<16, 1> vec(DotNoShuffle(A0, A1, C0, C1).v); + vec = vec.template Shuffle<{{ + 0,1, 4,5, + 8,9, 12,13, + 2,3, 6,7, + 10,11, 14,15 + }}>(); + return vec.v; + #endif + } + } + + + private: + constexpr static VectorF32<1, Packing*4> LengthNoShuffle( + VectorF32 A, + VectorF32 B, + VectorF32 C, + VectorF32 D + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + VectorF32<1, Packing*4> lenghtSq = LengthSqNoShuffle(A, B, C, D); + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32<1, Packing*4>(_mm_sqrt_ps(lenghtSq.v)); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32<1, Packing*4>(_mm256_sqrt_ps(lenghtSq.v)); + #ifdef __AVX512F__ + } else { + return VectorF32<1, Packing*4>(_mm512_sqrt_ps(lenghtSq.v)); + #endif + } + } + + constexpr static VectorF32<1, Packing*2> LengthNoShuffle( + VectorF32 A, + VectorF32 C + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + VectorF32<1, Packing*2> lenghtSq = LengthSqNoShuffle(A, C); + if constexpr(std::is_same_v::VectorType, __m128>) { + return VectorF32<1, Packing*2>(_mm_sqrt_ps(lenghtSq.v)); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + return VectorF32<1, Packing*2>(_mm256_sqrt_ps(lenghtSq.v)); + #ifdef __AVX512F__ + } else { + return VectorF32<1, Packing*2>(_mm512_sqrt_ps(lenghtSq.v)); + #endif + } + } + + constexpr static VectorF32<1, Packing*4> LengthSqNoShuffle( + VectorF32 A, + VectorF32 B, + VectorF32 C, + VectorF32 D + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + return DotNoShuffle(A, A, B, B, C, C, D, D); + } + + constexpr static VectorF32<1, Packing*2> LengthSqNoShuffle( + VectorF32 A, + VectorF32 C + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + return DotNoShuffle(A, A, C, C); + } + + + constexpr static VectorF32<1, Packing*4> DotNoShuffle( + VectorF32 A0, VectorF32 A1, + VectorF32 B0, VectorF32 B1, + VectorF32 C0, VectorF32 C1, + VectorF32 D0, VectorF32 D1 + ) requires(Len == 4 && Packing*Len == VectorBase::AlignmentElement) { + if constexpr(std::is_same_v::VectorType, __m128>) { + __m128 mulA = _mm_mul_ps(A0.v, A1.v); + __m128 mulB = _mm_mul_ps(B0.v, B1.v); + + __m128i row12Temp1 = _mm_unpacklo_epi32(_mm_castps_si128(mulA), _mm_castps_si128(mulB)); // A1 B1 A2 B2 + __m128i row34Temp1 = _mm_unpackhi_epi32(_mm_castps_si128(mulA), _mm_castps_si128(mulB)); // A3 B3 A4 B4 + + __m128 mulC = _mm_mul_ps(C0.v, C1.v); + __m128 mulD = _mm_mul_ps(D0.v, D1.v); + + __m128i row12Temp2 = _mm_unpacklo_epi32(_mm_castps_si128(mulC), _mm_castps_si128(mulD)); // C1 D1 C2 D2 + __m128i row34Temp2 = _mm_unpackhi_epi32(_mm_castps_si128(mulC), _mm_castps_si128(mulD)); // C3 D3 C4 D4 + + __m128 row1 = _mm_unpacklo_epi32(row12Temp1, row12Temp2); // A1 C1 B1 D1 + __m128 row2 = _mm_unpackhi_epi32(row12Temp1, row12Temp2); // A2 C2 B2 D2 + __m128 row3 = _mm_unpacklo_epi32(row34Temp1, row34Temp2); // A3 C3 B3 D3 + __m128 row4 = _mm_unpackhi_epi32(row34Temp1, row34Temp2); // A4 C4 B4 D4 + + + row1 = _mm_add_ps(row1, row2); + row1 = _mm_add_ps(row1, row3); + row1 = _mm_add_ps(row1, row4); + + return row1; + } else if constexpr(std::is_same_v::VectorType, __m256>) { + __m256 mulA = _mm256_mul_ps(A0.v, A1.v); + __m256 mulB = _mm256_mul_ps(B0.v, B1.v); + + __m256i row12Temp1 = _mm256_unpacklo_epi32(_mm256_castps_si256(mulA), _mm256_castps_si256(mulB)); // A1 B1 A2 B2 + __m256i row34Temp1 = _mm256_unpackhi_epi32(_mm256_castps_si256(mulA), _mm256_castps_si256(mulB)); // A3 B3 A4 B4 + + __m256 mulC = _mm256_mul_ps(C0.v, C1.v); + __m256 mulD = _mm256_mul_ps(D0.v, D1.v); + + __m256i row12Temp2 = _mm256_unpacklo_epi32(_mm256_castps_si256(mulC), _mm256_castps_si256(mulD)); // C1 D1 C2 D2 + __m256i row34Temp2 = _mm256_unpackhi_epi32(_mm256_castps_si256(mulC), _mm256_castps_si256(mulD)); // C3 D3 C4 D4 + + __m256 row1 = _mm256_unpacklo_epi32(row12Temp1, row12Temp2); // A1 C1 B1 D1 + __m256 row2 = _mm256_unpackhi_epi32(row12Temp1, row12Temp2); //A2 C2 B2 D2 + __m256 row3 = _mm256_unpacklo_epi32(row34Temp1, row34Temp2); // A3 C3 B3 D3 + __m256 row4 = _mm256_unpackhi_epi32(row34Temp1, row34Temp2); // A4 C4 B4 D4 + + row1 = _mm256_add_ps(row1, row2); + row1 = _mm256_add_ps(row1, row3); + row1 = _mm256_add_ps(row1, row4); + + return row1; + #ifdef __AVX512F__ + } else { + __m512 mulA = _mm512_mul_ps(A0.v, A1.v); + __m512 mulB = _mm512_mul_ps(B0.v, B1.v); + + __m512i row12Temp1 = _mm512_unpacklo_epi32(_mm512_castps_si512(mulA), _mm512_castps_si512(mulB)); // A1 B1 A2 B2 + __m512i row34Temp1 = _mm512_unpackhi_epi32(_mm512_castps_si512(mulA), _mm512_castps_si512(mulB)); // A3 B3 A4 B4 + + __m512 mulC = _mm512_mul_ps(C0.v, C1.v); + __m512 mulD = _mm512_mul_ps(D0.v, D1.v); + + __m512i row12Temp2 = _mm512_unpacklo_epi32(_mm512_castps_si512(mulC), _mm512_castps_si512(mulD)); // C1 D1 C2 D2 + __m512i row34Temp2 = _mm512_unpackhi_epi32(_mm512_castps_si512(mulC), _mm512_castps_si512(mulD)); // C3 D3 C4 D4 + + __m512 row1 = _mm512_unpacklo_epi32(row12Temp1, row12Temp2); // A1 C1 B1 D1 + __m512 row2 = _mm512_unpackhi_epi32(row12Temp1, row12Temp2); //A2 C2 B2 D2 + __m512 row3 = _mm512_unpacklo_epi32(row34Temp1, row34Temp2); // A3 C3 B3 D3 + __m512 row4 = _mm512_unpackhi_epi32(row34Temp1, row34Temp2); // A4 C4 B4 D4 + + row1 = _mm512_add_ps(row1, row2); + row1 = _mm512_add_ps(row1, row3); + row1 = _mm512_add_ps(row1, row4); + + return row1; + #endif + } + } + + constexpr static VectorF32<1, Packing*2> DotNoShuffle( + VectorF32 A0, VectorF32 A1, + VectorF32 C0, VectorF32 C1 + ) requires(Len == 2 && Packing*Len == VectorBase::AlignmentElement) { + if constexpr(std::is_same_v::VectorType, __m128>) { + __m128 mulA = _mm_mul_ps(A0.v, A1.v); + __m128 mulC = _mm_mul_ps(C0.v, C1.v); + __m128i row12Temp1 = _mm_unpacklo_epi32(_mm_castps_si128(mulA), _mm_castps_si128(mulC)); // A1 C1 A2 C2 + __m128i row56Temp1 = _mm_unpackhi_epi32(_mm_castps_si128(mulA), _mm_castps_si128(mulC)); // B1 D1 B2 D2 + __m128i row1TempTemp1 = row12Temp1; + __m128i row5TempTemp1 = row56Temp1; + + row12Temp1 = _mm_unpacklo_epi32(row12Temp1, row56Temp1); // A1 B1 C1 D1 + row56Temp1 = _mm_unpackhi_epi32(row1TempTemp1, row56Temp1); // A2 B2 C2 D2 + + return _mm_add_ps(row12Temp1, row56Temp1); + } else if constexpr(std::is_same_v::VectorType, __m256>) { + __m256 mulA = _mm256_mul_ps(A0.v, A1.v); + __m256 mulC = _mm256_mul_ps(C0.v, C1.v); + __m256i row12Temp1 = _mm256_unpacklo_epi32(_mm256_castps_si256(mulA), _mm256_castps_si256(mulC)); // A1 C1 A2 C2 + __m256i row56Temp1 = _mm256_unpackhi_epi32(_mm256_castps_si256(mulA), _mm256_castps_si256(mulC)); // B1 D1 B2 D2 + __m256i row1TempTemp1 = row12Temp1; + __m256i row5TempTemp1 = row56Temp1; + + row12Temp1 = _mm256_unpacklo_epi32(row12Temp1, row56Temp1); // A1 B1 C1 D1 + row56Temp1 = _mm256_unpackhi_epi32(row1TempTemp1, row56Temp1); // A2 B2 C2 D2 + + return _mm256_add_ps(row12Temp1, row56Temp1); + #ifdef __AVX512F__ + } else { + __m512 mulA = _mm512_mul_ps(A0.v, A1.v); + __m512 mulC = _mm512_mul_ps(C0.v, C1.v); + __m512i row12Temp1 = _mm512_unpacklo_epi32(_mm512_castps_si512(mulA), _mm512_castps_si512(mulC)); // A1 C1 A2 C2 + __m512i row56Temp1 = _mm512_unpackhi_epi32(_mm512_castps_si512(mulA), _mm512_castps_si512(mulC)); // B1 D1 B2 D2 + __m512i row1TempTemp1 = row12Temp1; + __m512i row5TempTemp1 = row56Temp1; + + row12Temp1 = _mm512_unpacklo_epi32(row12Temp1, row56Temp1); // A1 B1 C1 D1 + row56Temp1 = _mm512_unpackhi_epi32(row1TempTemp1, row56Temp1); // A2 B2 C2 D2 + + return _mm512_add_ps(row12Temp1, row56Temp1); + #endif + } + } + public: + + template ShuffleValues> + constexpr static VectorF32 Blend(VectorF32 a, VectorF32 b) { + constexpr auto mask = VectorBase::template GetBlendMaskEpi32(); + + if constexpr (std::is_same_v::VectorType, __m128>) { + return _mm_castsi128_ps(_mm_blend_epi32(_mm_castps_si128(a.v), _mm_castps_si128(b.v), mask)); + + } else if constexpr (std::is_same_v::VectorType, __m256>) { + return _mm256_castsi256_ps(_mm256_blend_epi32(_mm256_castps_si256(a.v), _mm256_castps_si256(b.v), mask)); + + #ifdef __AVX512F__ + } else if constexpr (std::is_same_v::VectorType, __m512>) { + return _mm512_castsi512_ps(_mm512_mask_blend_epi32(mask, _mm512_castps_si512(a.v), _mm512_castps_si512(b.v))); + #endif + + } + } + + constexpr static VectorF32 Rotate(VectorF32<3, Packing> v, VectorF32<4, Packing> q) requires(Len == 3) { + VectorF32<3, Packing> qv(q); + VectorF32 t = Cross(qv, v) * float(2); + return v + t * q.template Shuffle<{{3,3,3,3}}>() + Cross(qv, t); + } + + constexpr static VectorF32<4, 2> RotatePivot(VectorF32<3, Packing> v, VectorF32<4, Packing> q, VectorF32<3, Packing> pivot) requires(Len == 3) { + VectorF32 translated = v - pivot; + VectorF32<3, Packing> qv(q.v); + VectorF32 t = Cross(qv, translated) * float(2); + VectorF32 rotated = translated + t * q.template Shuffle<{{3,3,3,3}}>() + Cross(qv, t); + return rotated + pivot; + } + + constexpr static VectorF32<4, Packing> QuanternionFromEuler(VectorF32<3, Packing> EulerHalf) requires(Len == 4) { + std::tuple, VectorF32<3, Packing>> sinCos = EulerHalf.SinCos(); + VectorF32<4, Packing> sin = std::get<0>(sinCos); + VectorF32<4, Packing> cos = std::get<1>(sinCos); + + VectorF32<4, Packing> row1 = cos.template Shuffle<{{0,0,0,0}}>(); + row1 = Blend<{{0,1,1,1}}>(sin, row1); + + VectorF32<4, Packing> row2 = cos.template Shuffle<{{1,1,1,1}}>(); + row2 = Blend<{{1,0,1,1}}>(sin, row2); + + row1 *= row2; + + VectorF32<4, Packing> row3 = cos.template Shuffle<{{2,2,2,2}}>(); + row3 = Blend<{{1,1,0,1}}>(sin, row3); + + row1 *= row3; + + VectorF32<4, Packing> row4 = sin.template Shuffle<{{0,0,0,0}}>(); + row4 = Blend<{{0,1,1,1}}>(cos, row4); + + VectorF32<4, Packing> row5 = sin.template Shuffle<{{1,1,1,1}}>(); + row5 = Blend<{{1,0,1,1}}>(cos, row5); + + row4 *= row5; + + VectorF32<4, Packing> row6 = sin.template Shuffle<{{2,2,2,2}}>(); + row6 = Blend<{{1,1,0,1}}>(cos, row6); + row6 = row6.template Negate<{{true,false,true,false}}>(); + + row1 = MulitplyAdd(row4, row6, row1); + + return row1; + } + }; +#elif defined(__wasm_simd128__) + // WebAssembly SIMD128 implementation. VectorType is always v128_t and we + // cap Len*Packing*sizeof(float) at 16 bytes (i.e. up to 4 floats per + // vector) in Common.cppm so a single v128_t covers every instantiation. + // Operations without a direct SIMD equivalent (Shuffle with runtime indices, + // transcendentals, etc.) round-trip through a float[4] scratch buffer. + export template + struct VectorF32 : public VectorBase { + template + friend struct VectorF32; + using Base = VectorBase; + static constexpr std::uint8_t NElems = Base::AlignmentElement; + static_assert(NElems == 4, "WASM SIMD VectorF32 assumes 4-lane vectors"); + + constexpr VectorF32() = default; + constexpr VectorF32(v128_t vv) { this->v = vv; } + constexpr VectorF32(const float* vB) { Load(vB); } + constexpr VectorF32(float val) { this->v = wasm_f32x4_splat(val); } + + constexpr void Load(const float* vB) { this->v = wasm_v128_load(vB); } + constexpr void Store(float* vB) const { wasm_v128_store(vB, this->v); } + + template + constexpr std::array Store() const { + std::array r{}; + Store(r.data()); + return r; + } + + template + constexpr operator VectorF32() const { + alignas(16) float tmp[4]; + wasm_v128_store(tmp, this->v); + alignas(16) float out[4] = {0,0,0,0}; + const std::uint8_t copyLen = (BLen < Len) ? BLen : Len; + const std::uint8_t copyPack = (BPacking < Packing) ? BPacking : Packing; + for (std::uint8_t p = 0; p < copyPack; ++p) + for (std::uint8_t i = 0; i < copyLen; ++i) + out[p * BLen + i] = tmp[p * Len + i]; + return VectorF32(wasm_v128_load(out)); + } + + constexpr VectorF32 operator+(VectorF32 b) const { return VectorF32(wasm_f32x4_add(this->v, b.v)); } + constexpr VectorF32 operator-(VectorF32 b) const { return VectorF32(wasm_f32x4_sub(this->v, b.v)); } + constexpr VectorF32 operator*(VectorF32 b) const { return VectorF32(wasm_f32x4_mul(this->v, b.v)); } + constexpr VectorF32 operator/(VectorF32 b) const { return VectorF32(wasm_f32x4_div(this->v, b.v)); } + constexpr void operator+=(VectorF32 b) { this->v = wasm_f32x4_add(this->v, b.v); } + constexpr void operator-=(VectorF32 b) { this->v = wasm_f32x4_sub(this->v, b.v); } + constexpr void operator*=(VectorF32 b) { this->v = wasm_f32x4_mul(this->v, b.v); } + constexpr void operator/=(VectorF32 b) { this->v = wasm_f32x4_div(this->v, b.v); } + + constexpr VectorF32 operator+(float b) const { return *this + VectorF32(b); } + constexpr VectorF32 operator-(float b) const { return *this - VectorF32(b); } + constexpr VectorF32 operator*(float b) const { return *this * VectorF32(b); } + constexpr VectorF32 operator/(float b) const { return *this / VectorF32(b); } + constexpr void operator+=(float b) { *this += VectorF32(b); } + constexpr void operator-=(float b) { *this -= VectorF32(b); } + constexpr void operator*=(float b) { *this *= VectorF32(b); } + constexpr void operator/=(float b) { *this /= VectorF32(b); } + + constexpr VectorF32 operator-() const { return VectorF32(wasm_f32x4_neg(this->v)); } + + constexpr bool operator==(VectorF32 b) const { + return wasm_i32x4_bitmask(wasm_f32x4_eq(this->v, b.v)) == 0b1111; + } + constexpr bool operator!=(VectorF32 b) const { return !(*this == b); } + + template + constexpr VectorF32 ExtractLo() const { + alignas(16) float tmp[4]; wasm_v128_store(tmp, this->v); + alignas(16) float out[4] = {0,0,0,0}; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < ExtractLen; ++i) + out[p * ExtractLen + i] = tmp[p * Len + i]; + return VectorF32(wasm_v128_load(out)); + } + + constexpr VectorF32 Cos() const { + alignas(16) float tmp[4]; wasm_v128_store(tmp, this->v); + for (int i = 0; i < 4; ++i) tmp[i] = std::cos(tmp[i]); + return VectorF32(wasm_v128_load(tmp)); + } + constexpr VectorF32 Sin() const { + alignas(16) float tmp[4]; wasm_v128_store(tmp, this->v); + for (int i = 0; i < 4; ++i) tmp[i] = std::sin(tmp[i]); + return VectorF32(wasm_v128_load(tmp)); + } + constexpr std::tuple, VectorF32> SinCos() const { + return { Sin(), Cos() }; + } + + template values> + constexpr VectorF32 Negate() const { + constexpr auto mask = []() { + std::array m{}; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + m[p * Len + i] = values[i] ? 0x80000000u : 0u; + return m; + }(); + v128_t maskVec = wasm_v128_load(mask.data()); + return VectorF32(wasm_v128_xor(this->v, maskVec)); + } + + static constexpr VectorF32 MulitplyAdd(VectorF32 a, VectorF32 b, VectorF32 add) { + #ifdef __wasm_relaxed_simd__ + // Single-rounded FMA (a*b + c). Host-defined when FMA hardware is + // missing — accuracy may differ from the strict-SIMD wasm path. + return VectorF32(wasm_f32x4_relaxed_madd(a.v, b.v, add.v)); + #else + return VectorF32(wasm_f32x4_add(wasm_f32x4_mul(a.v, b.v), add.v)); + #endif + } + static constexpr VectorF32 MulitplySub(VectorF32 a, VectorF32 b, VectorF32 sub) { + #ifdef __wasm_relaxed_simd__ + // a*b - c is fused as madd(a, b, -c) — same op count as mul+sub + // but one rounding instead of two. + return VectorF32(wasm_f32x4_relaxed_madd(a.v, b.v, wasm_f32x4_neg(sub.v))); + #else + return VectorF32(wasm_f32x4_sub(wasm_f32x4_mul(a.v, b.v), sub.v)); + #endif + } + + constexpr static VectorF32 Cross(VectorF32 a, VectorF32 b) requires(Len == 3) { + v128_t a_yzx = wasm_i32x4_shuffle(a.v, a.v, 1, 2, 0, 3); + v128_t a_zxy = wasm_i32x4_shuffle(a.v, a.v, 2, 0, 1, 3); + v128_t b_yzx = wasm_i32x4_shuffle(b.v, b.v, 1, 2, 0, 3); + v128_t b_zxy = wasm_i32x4_shuffle(b.v, b.v, 2, 0, 1, 3); + #ifdef __wasm_relaxed_simd__ + // a_yzx*b_zxy - a_zxy*b_yzx fused as nmadd(a_zxy, b_yzx, a_yzx*b_zxy) + // = -(a_zxy*b_yzx) + a_yzx*b_zxy. Replaces a mul+sub pair with a + // single FMA. + return VectorF32(wasm_f32x4_relaxed_nmadd(a_zxy, b_yzx, wasm_f32x4_mul(a_yzx, b_zxy))); + #else + return VectorF32(wasm_f32x4_sub(wasm_f32x4_mul(a_yzx, b_zxy), wasm_f32x4_mul(a_zxy, b_yzx))); + #endif + } + + template ShuffleValues> + constexpr VectorF32 Shuffle() const { + alignas(16) float tmp[4]; wasm_v128_store(tmp, this->v); + alignas(16) float out[4] = {0,0,0,0}; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + out[p * Len + i] = tmp[p * Len + ShuffleValues[i]]; + return VectorF32(wasm_v128_load(out)); + } + + template ShuffleValues> + constexpr static VectorF32 Blend(VectorF32 a, VectorF32 b) { + constexpr auto mask = []() { + std::array m{}; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + m[p * Len + i] = ShuffleValues[i] ? 0xFFFFFFFFu : 0u; + return m; + }(); + v128_t maskVec = wasm_v128_load(mask.data()); + return VectorF32(wasm_v128_bitselect(b.v, a.v, maskVec)); + } + + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto LengthSq(VectorF32 first, Rest... rest) { + constexpr std::uint8_t N = VectorBase::BatchSize; + VectorF32<1, static_cast(Packing * N)> r; + std::array, N> args{ first, rest... }; + alignas(16) float buf[4] = {0,0,0,0}; + for (std::uint8_t i = 0; i < N; ++i) { + alignas(16) float tmp[4]; + wasm_v128_store(tmp, args[i].v); + for (std::uint8_t p = 0; p < Packing; ++p) { + float acc = 0.0f; + for (std::uint8_t k = 0; k < Len; ++k) { + float x = tmp[p * Len + k]; + acc += x * x; + } + buf[i * Packing + p] = acc; + } + } + r.v = wasm_v128_load(buf); + return r; + } + + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto Length(VectorF32 first, Rest... rest) { + auto sq = LengthSq(first, rest...); + sq.v = wasm_f32x4_sqrt(sq.v); + return sq; + } + + // Pairwise dot products packed into one v128. Only the first Len + // lanes contribute, so the same routine handles 3- and 4-component + // inputs — the 4th lane of Len==3 inputs may be garbage from Cross() + // and must not be summed. Takes BatchSize pairs (== 4 here since + // WASM AlignmentElement is always 4 and Packing must be 1). + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == 2 * VectorBase::BatchSize) && + (Len == 3 || Len == 4) && Packing == 1) + constexpr static VectorF32<1, 4> Dot(VectorF32 first, Rest... rest) { + constexpr std::uint8_t N = VectorBase::BatchSize; + std::array, 2 * N> args{ first, rest... }; + alignas(16) float out[4] = {0,0,0,0}; + for (std::uint8_t i = 0; i < N; ++i) { + alignas(16) float a[4], b[4]; + wasm_v128_store(a, args[2 * i].v); + wasm_v128_store(b, args[2 * i + 1].v); + for (std::uint8_t k = 0; k < Len; ++k) out[i] += a[k] * b[k]; + } + return VectorF32<1, 4>(wasm_v128_load(out)); + } + + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto Normalize(VectorF32 first, Rest... rest) { + auto normOne = [](VectorF32 u) { + alignas(16) float tmp[4]; wasm_v128_store(tmp, u.v); + alignas(16) float out[4] = {0,0,0,0}; + for (std::uint8_t p = 0; p < Packing; ++p) { + float acc = 0.0f; + for (std::uint8_t k = 0; k < Len; ++k) { + float x = tmp[p * Len + k]; + acc += x * x; + } + float invLen = acc > 0.0f ? 1.0f / std::sqrt(acc) : 0.0f; + for (std::uint8_t k = 0; k < Len; ++k) + out[p * Len + k] = tmp[p * Len + k] * invLen; + } + return VectorF32(wasm_v128_load(out)); + }; + return std::array, VectorBase::BatchSize>{ normOne(first), normOne(rest)... }; + } + + constexpr static VectorF32 Rotate(VectorF32<3, Packing> v, VectorF32<4, Packing> q) requires(Len == 3) { + alignas(16) float qBuf[4]; wasm_v128_store(qBuf, q.v); + alignas(16) float qvBuf[4] = {0,0,0,0}; + alignas(16) float qwBuf[4] = {0,0,0,0}; + for (std::uint8_t p = 0; p < Packing; ++p) { + qvBuf[p * 3 + 0] = qBuf[p * 4 + 0]; + qvBuf[p * 3 + 1] = qBuf[p * 4 + 1]; + qvBuf[p * 3 + 2] = qBuf[p * 4 + 2]; + for (std::uint8_t i = 0; i < 3; ++i) qwBuf[p * 3 + i] = qBuf[p * 4 + 3]; + } + VectorF32<3, Packing> qv(wasm_v128_load(qvBuf)); + VectorF32<3, Packing> qwBroadcast(wasm_v128_load(qwBuf)); + VectorF32<3, Packing> t = Cross(qv, v) * 2.0f; + return v + t * qwBroadcast + Cross(qv, t); + } + + constexpr static VectorF32<3, Packing> RotatePivot(VectorF32<3, Packing> v, VectorF32<4, Packing> q, VectorF32<3, Packing> pivot) requires(Len == 3) { + VectorF32<3, Packing> translated = v - pivot; + return Rotate(translated, q) + pivot; + } + + constexpr static VectorF32<4, Packing> QuanternionFromEuler(VectorF32<3, Packing> eulerHalf) requires(Len == 4) { + alignas(16) float eulerBuf[4]; wasm_v128_store(eulerBuf, eulerHalf.v); + alignas(16) float outBuf[4] = {0,0,0,0}; + for (std::uint8_t p = 0; p < Packing; ++p) { + float roll = eulerBuf[p * 3 + 0]; + float pitch = eulerBuf[p * 3 + 1]; + float yaw = eulerBuf[p * 3 + 2]; + float sr = std::sin(roll), cr = std::cos(roll); + float sp = std::sin(pitch), cp = std::cos(pitch); + float sy = std::sin(yaw), cy = std::cos(yaw); + outBuf[p * 4 + 0] = sr * cp * cy - cr * sp * sy; + outBuf[p * 4 + 1] = cr * sp * cy + sr * cp * sy; + outBuf[p * 4 + 2] = cr * cp * sy - sr * sp * cy; + outBuf[p * 4 + 3] = cr * cp * cy + sr * sp * sy; + } + return VectorF32<4, Packing>(wasm_v128_load(outBuf)); + } + }; +#elif defined(__riscv_vector) + // RISC-V V extension implementation. Storage is a GNU vector of 16/32/64 + // bytes (picked in Common.cppm from the guaranteed VLEN); native operators + // map to vfadd/vfsub/vfmul/vfdiv. Per-element loops compile to vrgather/ + // vmerge/vfsqrt when the autovectoriser can see the pattern, and to + // scalar fallback otherwise. Hand-tuned intrinsic paths + // (e.g. vsetvl + vfwmacc for batched dot) can land incrementally. + export template + struct VectorF32 : public VectorBase { + template + friend struct VectorF32; + using Base = VectorBase; + static constexpr std::uint8_t NElems = Base::AlignmentElement; + + constexpr VectorF32() = default; + constexpr VectorF32(typename Base::VectorType vv) { this->v = vv; } + constexpr VectorF32(const float* vB) { Load(vB); } + constexpr VectorF32(float val) { + for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = val; + } + + constexpr void Load(const float* vB) { + for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = vB[i]; + } + constexpr void Store(float* vB) const { + for (std::uint8_t i = 0; i < NElems; ++i) vB[i] = this->v[i]; + } + + template + constexpr std::array Store() const { + std::array r{}; + Store(r.data()); + return r; + } + + template + constexpr operator VectorF32() const { + VectorF32 r; + const std::uint8_t copyLen = (BLen < Len) ? BLen : Len; + const std::uint8_t copyPack = (BPacking < Packing) ? BPacking : Packing; + for (std::uint8_t p = 0; p < copyPack; ++p) + for (std::uint8_t i = 0; i < copyLen; ++i) + r.v[p * BLen + i] = this->v[p * Len + i]; + return r; + } + + constexpr VectorF32 operator+(VectorF32 b) const { return VectorF32(this->v + b.v); } + constexpr VectorF32 operator-(VectorF32 b) const { return VectorF32(this->v - b.v); } + constexpr VectorF32 operator*(VectorF32 b) const { return VectorF32(this->v * b.v); } + constexpr VectorF32 operator/(VectorF32 b) const { return VectorF32(this->v / b.v); } + constexpr void operator+=(VectorF32 b) { this->v = this->v + b.v; } + constexpr void operator-=(VectorF32 b) { this->v = this->v - b.v; } + constexpr void operator*=(VectorF32 b) { this->v = this->v * b.v; } + constexpr void operator/=(VectorF32 b) { this->v = this->v / b.v; } + + constexpr VectorF32 operator+(float b) const { return *this + VectorF32(b); } + constexpr VectorF32 operator-(float b) const { return *this - VectorF32(b); } + constexpr VectorF32 operator*(float b) const { return *this * VectorF32(b); } + constexpr VectorF32 operator/(float b) const { return *this / VectorF32(b); } + constexpr void operator+=(float b) { *this += VectorF32(b); } + constexpr void operator-=(float b) { *this -= VectorF32(b); } + constexpr void operator*=(float b) { *this *= VectorF32(b); } + constexpr void operator/=(float b) { *this /= VectorF32(b); } + + constexpr VectorF32 operator-() const { return VectorF32(-this->v); } + + constexpr bool operator==(VectorF32 b) const { + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + if (this->v[p * Len + i] != b.v[p * Len + i]) return false; + return true; + } + constexpr bool operator!=(VectorF32 b) const { return !(*this == b); } + + template + constexpr VectorF32 ExtractLo() const { + VectorF32 r; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < ExtractLen; ++i) + r.v[p * ExtractLen + i] = this->v[p * Len + i]; + return r; + } + + constexpr VectorF32 Cos() const { + VectorF32 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = std::cos(this->v[i]); + return r; + } + constexpr VectorF32 Sin() const { + VectorF32 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = std::sin(this->v[i]); + return r; + } + constexpr std::tuple, VectorF32> SinCos() const { + return { Sin(), Cos() }; + } + + template values> + constexpr VectorF32 Negate() const { + VectorF32 r; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + r.v[p * Len + i] = values[i] ? -this->v[p * Len + i] : this->v[p * Len + i]; + return r; + } + + // a*b + c — the compiler fuses to vfmacc.vv under the default + // -ffp-contract=on. No explicit intrinsic needed. + static constexpr VectorF32 MulitplyAdd(VectorF32 a, VectorF32 b, VectorF32 add) { + return VectorF32(a.v * b.v + add.v); + } + static constexpr VectorF32 MulitplySub(VectorF32 a, VectorF32 b, VectorF32 sub) { + return VectorF32(a.v * b.v - sub.v); + } + + constexpr static VectorF32 Cross(VectorF32 a, VectorF32 b) requires(Len == 3) { + VectorF32 r; + for (std::uint8_t p = 0; p < Packing; ++p) { + const std::uint8_t base = p * 3; + r.v[base + 0] = a.v[base + 1] * b.v[base + 2] - a.v[base + 2] * b.v[base + 1]; + r.v[base + 1] = a.v[base + 2] * b.v[base + 0] - a.v[base + 0] * b.v[base + 2]; + r.v[base + 2] = a.v[base + 0] * b.v[base + 1] - a.v[base + 1] * b.v[base + 0]; + } + return r; + } + + template ShuffleValues> + constexpr VectorF32 Shuffle() const { + VectorF32 r; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + r.v[p * Len + i] = this->v[p * Len + ShuffleValues[i]]; + return r; + } + + template ShuffleValues> + constexpr static VectorF32 Blend(VectorF32 a, VectorF32 b) { + VectorF32 r; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + r.v[p * Len + i] = ShuffleValues[i] ? b.v[p * Len + i] : a.v[p * Len + i]; + return r; + } + + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto LengthSq(VectorF32 first, Rest... rest) { + constexpr std::uint8_t N = VectorBase::BatchSize; + VectorF32<1, static_cast(Packing * N)> r; + std::array, N> args{ first, rest... }; + for (std::uint8_t i = 0; i < N; ++i) + for (std::uint8_t p = 0; p < Packing; ++p) { + float acc = 0.0f; + for (std::uint8_t k = 0; k < Len; ++k) { + float x = args[i].v[p * Len + k]; + acc += x * x; + } + r.v[i * Packing + p] = acc; + } + return r; + } + + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto Length(VectorF32 first, Rest... rest) { + auto sq = LengthSq(first, rest...); + for (std::uint8_t i = 0; i < decltype(sq)::NElems; ++i) sq.v[i] = std::sqrt(sq.v[i]); + return sq; + } + + // Pairwise dot products across BatchSize pairs. The 4th lane of Len==3 + // inputs may carry garbage from Cross(), so only the first Len lanes + // are summed per pair. + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == 2 * VectorBase::BatchSize)) + constexpr static auto Dot(VectorF32 first, Rest... rest) { + constexpr std::uint8_t N = VectorBase::BatchSize; + VectorF32<1, static_cast(Packing * N)> r; + std::array, 2 * N> args{ first, rest... }; + for (std::uint8_t i = 0; i < N; ++i) + for (std::uint8_t p = 0; p < Packing; ++p) { + float acc = 0.0f; + for (std::uint8_t k = 0; k < Len; ++k) + acc += args[2 * i].v[p * Len + k] * args[2 * i + 1].v[p * Len + k]; + r.v[i * Packing + p] = acc; + } + return r; + } + + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto Normalize(VectorF32 first, Rest... rest) { + auto normOne = [](VectorF32 u) { + VectorF32 out; + for (std::uint8_t p = 0; p < Packing; ++p) { + float acc = 0.0f; + for (std::uint8_t k = 0; k < Len; ++k) { + float x = u.v[p * Len + k]; + acc += x * x; + } + float invLen = acc > 0.0f ? 1.0f / std::sqrt(acc) : 0.0f; + for (std::uint8_t k = 0; k < Len; ++k) + out.v[p * Len + k] = u.v[p * Len + k] * invLen; + } + return out; + }; + return std::array, VectorBase::BatchSize>{ normOne(first), normOne(rest)... }; + } + + constexpr static VectorF32 Rotate(VectorF32<3, Packing> v, VectorF32<4, Packing> q) requires(Len == 3) { + VectorF32<3, Packing> qv; + VectorF32<3, Packing> qwBroadcast; + for (std::uint8_t p = 0; p < Packing; ++p) { + qv.v[p * 3 + 0] = q.v[p * 4 + 0]; + qv.v[p * 3 + 1] = q.v[p * 4 + 1]; + qv.v[p * 3 + 2] = q.v[p * 4 + 2]; + for (std::uint8_t i = 0; i < 3; ++i) qwBroadcast.v[p * 3 + i] = q.v[p * 4 + 3]; + } + VectorF32<3, Packing> t = Cross(qv, v) * 2.0f; + return v + t * qwBroadcast + Cross(qv, t); + } + + constexpr static VectorF32<3, Packing> RotatePivot(VectorF32<3, Packing> v, VectorF32<4, Packing> q, VectorF32<3, Packing> pivot) requires(Len == 3) { + VectorF32<3, Packing> translated = v - pivot; + return Rotate(translated, q) + pivot; + } + + constexpr static VectorF32<4, Packing> QuanternionFromEuler(VectorF32<3, Packing> eulerHalf) requires(Len == 4) { + VectorF32<4, Packing> r; + for (std::uint8_t p = 0; p < Packing; ++p) { + float roll = eulerHalf.v[p * 3 + 0]; + float pitch = eulerHalf.v[p * 3 + 1]; + float yaw = eulerHalf.v[p * 3 + 2]; + float sr = std::sin(roll), cr = std::cos(roll); + float sp = std::sin(pitch), cp = std::cos(pitch); + float sy = std::sin(yaw), cy = std::cos(yaw); + r.v[p * 4 + 0] = sr * cp * cy - cr * sp * sy; + r.v[p * 4 + 1] = cr * sp * cy + sr * cp * sy; + r.v[p * 4 + 2] = cr * cp * sy - sr * sp * cy; + r.v[p * 4 + 3] = cr * cp * cy + sr * sp * sy; + } + return r; + } + }; +#else + // Scalar software fallback for non-x86_64 targets. Future arches can swap + // in their own intrinsic implementation by adding an arch-specific branch + // above and gating this one out. + export template + struct VectorF32 : public VectorBase { + template + friend struct VectorF32; + using Base = VectorBase; + static constexpr std::uint8_t NElems = Base::AlignmentElement; + + constexpr VectorF32() = default; + constexpr VectorF32(typename Base::VectorType vv) { + this->v = vv; + } + constexpr VectorF32(const float* vB) { Load(vB); } +#ifdef __FLT16_MAX__ + constexpr VectorF32(const _Float16* vB) { Load(vB); } +#endif + constexpr VectorF32(float val) { + for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = val; + } + + constexpr void Load(const float* vB) { + for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = vB[i]; + } + constexpr void Store(float* vB) const { + for (std::uint8_t i = 0; i < NElems; ++i) vB[i] = this->v[i]; + } +#ifdef __FLT16_MAX__ + constexpr void Load(const _Float16* vB) { + for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = static_cast(vB[i]); + } + constexpr void Store(_Float16* vB) const { + for (std::uint8_t i = 0; i < NElems; ++i) vB[i] = static_cast<_Float16>(this->v[i]); + } +#endif + + template + constexpr std::array Store() const { + std::array r{}; + Store(r.data()); + return r; + } + + template + constexpr operator VectorF32() const { + VectorF32 r; + const std::uint8_t copyLen = (BLen < Len) ? BLen : Len; + const std::uint8_t copyPack = (BPacking < Packing) ? BPacking : Packing; + for (std::uint8_t p = 0; p < copyPack; ++p) + for (std::uint8_t i = 0; i < copyLen; ++i) + r.v[p * BLen + i] = this->v[p * Len + i]; + return r; + } + + constexpr VectorF32 operator+(VectorF32 b) const { + VectorF32 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] + b.v[i]; + return r; + } + constexpr VectorF32 operator-(VectorF32 b) const { + VectorF32 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] - b.v[i]; + return r; + } + constexpr VectorF32 operator*(VectorF32 b) const { + VectorF32 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] * b.v[i]; + return r; + } + constexpr VectorF32 operator/(VectorF32 b) const { + VectorF32 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] / b.v[i]; + return r; + } + + constexpr void operator+=(VectorF32 b) { for (std::uint8_t i=0;iv[i] += b.v[i]; } + constexpr void operator-=(VectorF32 b) { for (std::uint8_t i=0;iv[i] -= b.v[i]; } + constexpr void operator*=(VectorF32 b) { for (std::uint8_t i=0;iv[i] *= b.v[i]; } + constexpr void operator/=(VectorF32 b) { for (std::uint8_t i=0;iv[i] /= b.v[i]; } + + constexpr VectorF32 operator+(float b) const { return *this + VectorF32(b); } + constexpr VectorF32 operator-(float b) const { return *this - VectorF32(b); } + constexpr VectorF32 operator*(float b) const { return *this * VectorF32(b); } + constexpr VectorF32 operator/(float b) const { return *this / VectorF32(b); } + constexpr void operator+=(float b) { *this += VectorF32(b); } + constexpr void operator-=(float b) { *this -= VectorF32(b); } + constexpr void operator*=(float b) { *this *= VectorF32(b); } + constexpr void operator/=(float b) { *this /= VectorF32(b); } + + constexpr VectorF32 operator-() const { + VectorF32 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = -this->v[i]; + return r; + } + + constexpr bool operator==(VectorF32 b) const { + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + if (this->v[p * Len + i] != b.v[p * Len + i]) return false; + return true; + } + constexpr bool operator!=(VectorF32 b) const { return !(*this == b); } + + template + constexpr VectorF32 ExtractLo() const { + VectorF32 r; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < ExtractLen; ++i) + r.v[p * ExtractLen + i] = this->v[p * Len + i]; + return r; + } + + constexpr VectorF32 Cos() const { + VectorF32 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = std::cos(this->v[i]); + return r; + } + constexpr VectorF32 Sin() const { + VectorF32 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = std::sin(this->v[i]); + return r; + } + constexpr std::tuple, VectorF32> SinCos() const { + return { Sin(), Cos() }; + } + + template values> + constexpr VectorF32 Negate() const { + VectorF32 r; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + r.v[p * Len + i] = values[i] ? -this->v[p * Len + i] : this->v[p * Len + i]; + return r; + } + + static constexpr VectorF32 MulitplyAdd(VectorF32 a, VectorF32 b, VectorF32 add) { + VectorF32 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = a.v[i] * b.v[i] + add.v[i]; + return r; + } + static constexpr VectorF32 MulitplySub(VectorF32 a, VectorF32 b, VectorF32 sub) { + VectorF32 r; + for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = a.v[i] * b.v[i] - sub.v[i]; + return r; + } + + constexpr static VectorF32 Cross(VectorF32 a, VectorF32 b) requires(Len == 3) { + VectorF32 r; + for (std::uint8_t p = 0; p < Packing; ++p) { + const std::uint8_t base = p * 3; + r.v[base + 0] = a.v[base + 1] * b.v[base + 2] - a.v[base + 2] * b.v[base + 1]; + r.v[base + 1] = a.v[base + 2] * b.v[base + 0] - a.v[base + 0] * b.v[base + 2]; + r.v[base + 2] = a.v[base + 0] * b.v[base + 1] - a.v[base + 1] * b.v[base + 0]; + } + return r; + } + + template ShuffleValues> + constexpr VectorF32 Shuffle() const { + VectorF32 r; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + r.v[p * Len + i] = this->v[p * Len + ShuffleValues[i]]; + return r; + } + + template ShuffleValues> + constexpr static VectorF32 Blend(VectorF32 a, VectorF32 b) { + VectorF32 r; + for (std::uint8_t p = 0; p < Packing; ++p) + for (std::uint8_t i = 0; i < Len; ++i) + r.v[p * Len + i] = ShuffleValues[i] ? b.v[p * Len + i] : a.v[p * Len + i]; + return r; + } + + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto LengthSq(VectorF32 first, Rest... rest) { + constexpr std::uint8_t N = VectorBase::BatchSize; + VectorF32<1, static_cast(Packing * N)> r; + std::array, N> args{ first, rest... }; + for (std::uint8_t i = 0; i < N; ++i) + for (std::uint8_t p = 0; p < Packing; ++p) { + float acc = 0.0f; + for (std::uint8_t k = 0; k < Len; ++k) { + float x = args[i].v[p * Len + k]; + acc += x * x; + } + r.v[i * Packing + p] = acc; + } + return r; + } + + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto Length(VectorF32 first, Rest... rest) { + auto sq = LengthSq(first, rest...); + for (std::uint8_t i = 0; i < decltype(sq)::NElems; ++i) sq.v[i] = std::sqrt(sq.v[i]); + return sq; + } + + template + requires((std::is_same_v> && ...) && + (1 + sizeof...(Rest) == VectorBase::BatchSize)) + constexpr static auto Normalize(VectorF32 first, Rest... rest) { + auto normOne = [](VectorF32 u) { + VectorF32 out; + for (std::uint8_t p = 0; p < Packing; ++p) { + float acc = 0.0f; + for (std::uint8_t k = 0; k < Len; ++k) { + float x = u.v[p * Len + k]; + acc += x * x; + } + float invLen = acc > 0.0f ? 1.0f / std::sqrt(acc) : 0.0f; + for (std::uint8_t k = 0; k < Len; ++k) + out.v[p * Len + k] = u.v[p * Len + k] * invLen; + } + return out; + }; + return std::array, VectorBase::BatchSize>{ normOne(first), normOne(rest)... }; + } + + constexpr static VectorF32 Rotate(VectorF32<3, Packing> v, VectorF32<4, Packing> q) requires(Len == 3) { + VectorF32<3, Packing> qv; + VectorF32<3, Packing> qwBroadcast; + for (std::uint8_t p = 0; p < Packing; ++p) { + qv.v[p * 3 + 0] = q.v[p * 4 + 0]; + qv.v[p * 3 + 1] = q.v[p * 4 + 1]; + qv.v[p * 3 + 2] = q.v[p * 4 + 2]; + for (std::uint8_t i = 0; i < 3; ++i) qwBroadcast.v[p * 3 + i] = q.v[p * 4 + 3]; + } + VectorF32<3, Packing> t = Cross(qv, v) * 2.0f; + return v + t * qwBroadcast + Cross(qv, t); + } + + constexpr static VectorF32<3, Packing> RotatePivot(VectorF32<3, Packing> v, VectorF32<4, Packing> q, VectorF32<3, Packing> pivot) requires(Len == 3) { + VectorF32<3, Packing> translated = v - pivot; + return Rotate(translated, q) + pivot; + } + + constexpr static VectorF32<4, Packing> QuanternionFromEuler(VectorF32<3, Packing> eulerHalf) requires(Len == 4) { + VectorF32<4, Packing> r; + for (std::uint8_t p = 0; p < Packing; ++p) { + float roll = eulerHalf.v[p * 3 + 0]; + float pitch = eulerHalf.v[p * 3 + 1]; + float yaw = eulerHalf.v[p * 3 + 2]; + float sr = std::sin(roll), cr = std::cos(roll); + float sp = std::sin(pitch), cp = std::cos(pitch); + float sy = std::sin(yaw), cy = std::cos(yaw); + r.v[p * 4 + 0] = sr * cp * cy - cr * sp * sy; + r.v[p * 4 + 1] = cr * sp * cy + sr * cp * sy; + r.v[p * 4 + 2] = cr * cp * sy - sr * sp * cy; + r.v[p * 4 + 3] = cr * cp * cy + sr * sp * sy; + } + return r; + } + }; +#endif +} + +export template +struct std::formatter> : std::formatter { + constexpr auto format(const Crafter::VectorF32& obj, format_context& ctx) const { + std::array::AlignmentElement> vec = obj.template Store(); + std::string out = "{"; + for(std::uint32_t i = 0; i < Packing; i++) { + out += "{"; + for(std::uint32_t i2 = 0; i2 < Len; i2++) { + out += std::format("{}", static_cast(vec[i * Len + i2])); + if (i2 + 1 < Len) out += ","; + } + out += "}"; + } + out += "}"; + return std::formatter::format(out, ctx); + } }; \ No newline at end of file