Catcrafts/Crafter.Math
Watch
2
0
Fork
You've already forked Crafter.Math
 0
Code Issues 1 Pull requests Projects Releases Packages Wiki Activity Actions
Crafter.Math/interfaces/Crafter.Math-VectorF16.cppm
Jorijn van der Graaf 0c55e76873
 ARM NEON 
NEON backend for VectorF32 on AArch64: one float32x4_t per vector
(Len * Packing <= 4), with a vectorised sin/cos port of the SSE helpers.

VectorF16 gets a native float16x8_t path when the target has ARMv8.2-FP16
arithmetic. Without it VectorF16 aliases VectorF32, which converts
fp16 <-> fp32 on Load/Store.

The scalar fallback gains the batched Dot that MatrixRowMajor relies on.
Test tiers added for armv8-a and armv8.2-a+fp16.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-06 17:13:54 +02:00

1701 lines
No EOL
98 KiB
C++
Executable file
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

//SPDX-License-Identifier: LGPL-3.0-only
//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
module;
#ifdef __x86_64
#include <immintrin.h>
#endif
#if defined(__aarch64__) && defined(__ARM_NEON)
#include <arm_neon.h>
#endif
export module Crafter.Math:VectorF16;
import std;
import :Common;
// A real VectorF16 struct is provided only when _Float16 is available AND we
// are on an x86_64 build with AVX512FP16 or an AArch64 NEON build with
// ARMv8.2-FP16 (intrinsic impls below), or on a target with neither x86_64 nor
// NEON (scalar fallback further below). On x86_64 without AVX512FP16, on
// AArch64 NEON without FP16 or anywhere without _Float16, Crafter.Math:Basic
// aliases VectorF16 to VectorF32 instead.
#ifdef __AVX512FP16__
namespace Crafter {
export template <std::uint8_t Len, std::uint8_t Packing>
struct VectorF16 : public VectorBase<Len, Packing, _Float16> {
template <std::uint8_t Len2, std::uint8_t Packing2>
friend struct VectorF16;
constexpr VectorF16() = default;
constexpr VectorF16(VectorBase<Len, Packing, _Float16>::VectorType v) {
this->v = v;
}
constexpr VectorF16(const _Float16* vB) {
Load(vB);
};
constexpr VectorF16(_Float16 val) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
this->v = _mm_set1_ph(val);
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
this->v = _mm256_set1_ph(val);
} else {
this->v = _mm512_set1_ph(val);
}
};
constexpr void Load(const _Float16* vB) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
this->v = _mm_loadu_ph(vB);
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
this->v = _mm256_loadu_ph(vB);
} else {
this->v = _mm512_loadu_ph(vB);
}
}
constexpr void Store(_Float16* vB) const {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
_mm_storeu_ph(vB, this->v);
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
_mm256_storeu_ph(vB, this->v);
} else {
_mm512_storeu_ph(vB, this->v);
}
}
template<typename T>
constexpr std::array<_Float16, VectorBase<Len, Packing, _Float16>::AlignmentElement> Store() const {
std::array<_Float16, VectorBase<Len, Packing, _Float16>::AlignmentElement> returnArray;
Store(returnArray.data());
return returnArray;
}
template <std::uint8_t BLen, std::uint8_t BPacking>
constexpr operator VectorF16<BLen, BPacking>() const {
if constexpr (Len == BLen) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h> && std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<BLen, BPacking>(_mm256_castph256_ph128(this->v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m512h> && std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<BLen, BPacking>(_mm512_castph512_ph128(this->v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m512h> && std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<BLen, BPacking>(_mm512_castph512_ph256(this->v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h> && std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<BLen, BPacking>(_mm256_castph128_ph256(this->v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h> && std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m512h>) {
return VectorF16<BLen, BPacking>(_mm512_castph128_ph512(this->v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h> && std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m512h>) {
return VectorF16<BLen, BPacking>(_mm512_castph256_ph512(this->v));
} else {
return VectorF16<BLen, BPacking>(this->v);
}
} else if constexpr (BLen <= Len) {
return this->template ExtractLo<BLen>();
} else {
if constexpr(std::is_same_v<typename VectorBase<BLen, BPacking, _Float16>::VectorType, __m128h>) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
constexpr std::array<std::uint8_t, VectorBase<Len, Packing, _Float16>::Alignment> shuffleMask = VectorBase<Len, Packing, _Float16>::template GetExtractLoMaskEpi8<BLen>();
__m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data());
return VectorF16<BLen, BPacking>(_mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(this->v), shuffleVec)));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
constexpr std::array<std::uint16_t, VectorBase<Len, Packing, _Float16>::AlignmentElement> permMask =VectorBase<Len, Packing, _Float16>::template GetExtractLoMaskEpi16<BLen>();
__m256i permIdx = _mm256_loadu_epi16(permMask.data());
__m256i result = _mm256_permutexvar_epi16(permIdx, _mm_castph_si256(this->v));
return VectorF16<BLen, BPacking>(_mm_castsi128_ph(_mm256_castsi256_si128(result)));
} else {
constexpr std::array<std::uint16_t, VectorBase<Len, Packing, _Float16>::AlignmentElement> permMask = VectorBase<Len, Packing, _Float16>::template GetExtractLoMaskEpi16<BLen>();
__m512i permIdx = _mm512_loadu_epi16(permMask.data());
__m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castph_si512(this->v));
return VectorF16<BLen, BPacking>(_mm_castsi128_ph(_mm512_castsi512_si128(result)));
}
} else if constexpr(std::is_same_v<typename VectorBase<BLen, BPacking, _Float16>::VectorType, __m256h>) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
constexpr std::array<std::uint16_t,VectorBase<BLen, Packing, _Float16>::AlignmentElement> permMask = VectorBase<BLen, Packing, _Float16>::template GetExtractLoMaskEpi16<BLen>();
__m256i permIdx = _mm256_loadu_epi16(permMask.data());
__m256i result = _mm256_permutexvar_epi16(permIdx, _mm256_castsi128_si256(_mm_castph_si128(this->v)));
return VectorF16<BLen, BPacking>(_mm256_castsi256_ph(result));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
constexpr std::array<std::uint16_t,VectorBase<BLen, Packing, _Float16>::AlignmentElement> permMask = VectorBase<BLen, Packing, _Float16>::template GetExtractLoMaskEpi16<BLen>();
__m256i permIdx = _mm256_loadu_epi16(permMask.data());
__m256i result = _mm256_permutexvar_epi16(permIdx, _mm256_castph_si256(this->v));
return VectorF16<BLen, BPacking>(_mm256_castsi256_ph(result));
} else {
constexpr std::array<std::uint16_t,VectorBase<BLen, Packing, _Float16>::AlignmentElement> permMask = VectorBase<BLen, Packing, _Float16>::template GetExtractLoMaskEpi16<BLen>();
__m256i permIdx = _mm512_loadu_epi16(permMask.data());
__m256i result = _mm512_permutexvar_epi16(permIdx, _mm512_castsi512_si256(_mm512_castph_si512(this->v)));
return VectorF16<BLen, BPacking>(_mm256_castsi256_ph(result));
}
} else {
if constexpr(std::is_same_v<typename VectorBase<BLen, BPacking, _Float16>::VectorType, __m128h>) {
constexpr std::array<std::uint16_t,VectorBase<BLen, Packing, _Float16>::AlignmentElement> permMask = VectorBase<BLen, Packing, _Float16>::template GetExtractLoMaskEpi16<BLen>();
__m512i permIdx = _mm512_loadu_epi16(permMask.data());
__m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castsi128_si512(_mm_castph_si128(this->v)));
return VectorF16<BLen, BPacking>(_mm512_castsi512_ph(result));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
constexpr std::array<std::uint16_t,VectorBase<BLen, Packing, _Float16>::AlignmentElement> permMask = VectorBase<BLen, Packing, _Float16>::template GetExtractLoMaskEpi16<BLen>();
__m512i permIdx = _mm512_loadu_epi16(permMask.data());
__m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castsi256_si512(_mm256_castph_si256(this->v)));
return VectorF16<BLen, BPacking>(_mm512_castsi512_ph(result));
} else {
constexpr std::array<std::uint16_t,VectorBase<BLen, Packing, _Float16>::AlignmentElement> permMask = VectorBase<BLen, Packing, _Float16>::template GetExtractLoMaskEpi16<BLen>();
__m512i permIdx = _mm512_loadu_epi16(permMask.data());
__m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castph_si512(this->v));
return VectorF16<BLen, BPacking>(_mm512_castsi512_ph(result));
}
}
}
}
constexpr VectorF16<Len, Packing> operator+(VectorF16<Len, Packing> b) const {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<Len, Packing>(_mm_add_ph(this->v, b.v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<Len, Packing>(_mm256_add_ph(this->v, b.v));
} else {
return VectorF16<Len, Packing>(_mm512_add_ph(this->v, b.v));
}
}
constexpr VectorF16<Len, Packing> operator-(VectorF16<Len, Packing> b) const {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<Len, Packing>(_mm_sub_ph(this->v, b.v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<Len, Packing>(_mm256_sub_ph(this->v, b.v));
} else {
return VectorF16<Len, Packing>(_mm512_sub_ph(this->v, b.v));
}
}
constexpr VectorF16<Len, Packing> operator*(VectorF16<Len, Packing> b) const {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<Len, Packing>(_mm_mul_ph(this->v, b.v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<Len, Packing>(_mm256_mul_ph(this->v, b.v));
} else {
return VectorF16<Len, Packing>(_mm512_mul_ph(this->v, b.v));
}
}
constexpr VectorF16<Len, Packing> operator/(VectorF16<Len, Packing> b) const {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<Len, Packing>(_mm_div_ph(this->v, b.v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<Len, Packing>(_mm256_div_ph(this->v, b.v));
} else {
return VectorF16<Len, Packing>(_mm512_div_ph(this->v, b.v));
}
}
constexpr void operator+=(VectorF16<Len, Packing> b) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
this->v = _mm_add_ph(this->v, b.v);
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
this->v = _mm256_add_ph(this->v, b.v);
} else {
this->v = _mm512_add_ph(this->v, b.v);
}
}
constexpr void operator-=(VectorF16<Len, Packing> b) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
this->v = _mm_sub_ph(this->v, b.v);
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
this->v = _mm256_sub_ph(this->v, b.v);
} else {
this->v = _mm512_sub_ph(this->v, b.v);
}
}
constexpr void operator*=(VectorF16<Len, Packing> b) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
this->v = _mm_mul_ph(this->v, b.v);
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
this->v = _mm256_mul_ph(this->v, b.v);
} else {
this->v = _mm512_mul_ph(this->v, b.v);
}
}
constexpr void operator/=(VectorF16<Len, Packing> b) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
this->v = _mm_div_ph(this->v, b.v);
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
this->v = _mm256_div_ph(this->v, b.v);
} else {
this->v = _mm512_div_ph(this->v, b.v);
}
}
constexpr VectorF16<Len, Packing> operator+(_Float16 b) {
VectorF16<Len, Packing> vB(b);
return *this + vB;
}
constexpr VectorF16<Len, Packing> operator-(_Float16 b) {
VectorF16<Len, Packing> vB(b);
return *this - vB;
}
constexpr VectorF16<Len, Packing> operator*(_Float16 b) {
VectorF16<Len, Packing> vB(b);
return *this * vB;
}
constexpr VectorF16<Len, Packing> operator/(_Float16 b) {
VectorF16<Len, Packing> vB(b);
return *this / vB;
}
constexpr void operator+=(_Float16 b) {
VectorF16<Len, Packing> vB(b);
*this += vB;
}
constexpr void operator-=(_Float16 b) {
VectorF16<Len, Packing> vB(b);
*this -= vB;
}
constexpr void operator*=(_Float16 b) {
VectorF16<Len, Packing> vB(b);
*this *= vB;
}
constexpr void operator/=(_Float16 b) {
VectorF16<Len, Packing> vB(b);
*this /= vB;
}
constexpr VectorF16<Len, Packing> operator-(){
return Negate<VectorBase<Len, Packing, _Float16>::GetAllTrue()>();
}
constexpr bool operator==(VectorF16<Len, Packing> b) const {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return _mm_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) == 255;
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return _mm256_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) == 65535;
} else {
return _mm512_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) == 4294967295;
}
}
constexpr bool operator!=(VectorF16<Len, Packing> b) const {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return _mm_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) != 255;
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return _mm256_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) != 65535;
} else {
return _mm512_cmp_ph_mask(this->v, b.v, _CMP_EQ_OQ) != 4294967295;
}
}
template<std::uint32_t ExtractLen>
constexpr VectorF16<ExtractLen, Packing> ExtractLo() const {
if constexpr(Packing > 1) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
constexpr std::array<std::uint8_t,VectorBase<Len, Packing, _Float16>::Alignment> shuffleMask = VectorBase<Len, Packing, _Float16>::template GetExtractLoMaskEpi8<ExtractLen>();
__m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data());
return VectorF16<ExtractLen, Packing>(_mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(this->v), shuffleVec)));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
constexpr std::array<std::uint16_t,VectorBase<Len, Packing, _Float16>::AlignmentElement> permMask = VectorBase<Len, Packing, _Float16>::template GetExtractLoMaskEpi16<ExtractLen>();
__m256i permIdx = _mm256_loadu_epi16(permMask.data());
__m256i result = _mm256_permutexvar_epi16(permIdx, _mm256_castph_si256(this->v));
if constexpr(std::is_same_v<typename VectorBase<ExtractLen, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<ExtractLen, Packing>(_mm256_castph256_ph128(_mm256_castsi256_ph(result)));
} else {
return VectorF16<ExtractLen, Packing>(_mm256_castsi256_ph(result));
}
} else {
constexpr std::array<std::uint16_t, VectorBase<Len, Packing, _Float16>::AlignmentElement> permMask = VectorBase<Len, Packing, _Float16>::template GetExtractLoMaskEpi16<ExtractLen>();
__m512i permIdx = _mm512_loadu_epi16(permMask.data());
__m512i result = _mm512_permutexvar_epi16(permIdx, _mm512_castph_si512(this->v));
if constexpr(std::is_same_v<typename VectorBase<ExtractLen, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<ExtractLen, Packing>(_mm512_castph512_ph128(_mm512_castsi512_ph(result)));
} else if constexpr(std::is_same_v<typename VectorBase<ExtractLen, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<ExtractLen, Packing>(_mm512_castph512_ph256(_mm512_castsi512_ph(result)));
} else {
return VectorF16<ExtractLen, Packing>(_mm512_castsi512_ph(result));
}
}
} else {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h> && std::is_same_v<typename VectorBase<ExtractLen, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<ExtractLen, Packing>(_mm256_castph256_ph128(this->v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m512h> && std::is_same_v<typename VectorBase<ExtractLen, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<ExtractLen, Packing>(_mm512_castph512_ph128(this->v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m512h> && std::is_same_v<typename VectorBase<ExtractLen, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<ExtractLen, Packing>(_mm512_castph512_ph256(this->v));
} else {
return VectorF16<ExtractLen, Packing>(this->v);
}
}
}
constexpr VectorF16<Len, Packing> Cos() {
if constexpr (std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
__m256 wide = _mm256_cvtph_ps(_mm_castph_si128(this->v));
wide = VectorBase<Len, Packing, _Float16>::cos_f32x8(wide);
return VectorF16<Len, Packing>(
_mm_castsi128_ph(_mm256_cvtps_ph(wide, _MM_FROUND_TO_NEAREST_INT)));
} else if constexpr (std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
__m512 wide = _mm512_cvtph_ps(_mm256_castph_si256(this->v));
wide = VectorBase<Len, Packing, _Float16>::cos_f32x16(wide);
return VectorF16<Len, Packing>(
_mm256_castsi256_ph(_mm512_cvtps_ph(wide, _MM_FROUND_TO_NEAREST_INT)));
} else {
__m256i lo = _mm512_castsi512_si256(_mm512_castph_si512(this->v));
__m256i hi = _mm512_extracti64x4_epi64(_mm512_castph_si512(this->v), 1);
__m512i cosLo =VectorBase<Len, Packing, _Float16>::cos_f32x16(_mm512_cvtph_ps(lo));
__m512i cosHi =VectorBase<Len, Packing, _Float16>::cos_f32x16(_mm512_cvtph_ps(hi));
__m256i lo_ph = _mm512_cvtps_ph(cosLo, _MM_FROUND_TO_NEAREST_INT);
__m256i hi_ph = _mm512_cvtps_ph(cosHi, _MM_FROUND_TO_NEAREST_INT);
return VectorF16<Len, Packing>(_mm512_castsi512_ph(_mm512_inserti64x4(_mm512_castsi256_si512(lo_ph), hi_ph, 1)));
}
}
constexpr VectorF16<Len, Packing> Sin() {
if constexpr (std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
__m256 wide = _mm256_cvtph_ps(_mm_castph_si128(this->v));
wide = VectorBase<Len, Packing, _Float16>::sin_f32x8(wide);
return VectorF16<Len, Packing>(_mm_castsi128_ph(_mm256_cvtps_ph(wide, _MM_FROUND_TO_NEAREST_INT)));
} else if constexpr (std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
__m512 wide = _mm512_cvtph_ps(_mm256_castph_si256(this->v));
wide = VectorBase<Len, Packing, _Float16>::sin_f32x16(wide);
return VectorF16<Len, Packing>(_mm256_castsi256_ph(_mm512_cvtps_ph(wide, _MM_FROUND_TO_NEAREST_INT)));
} else {
__m256i lo = _mm512_castsi512_si256(_mm512_castph_si512(this->v));
__m256i hi = _mm512_extracti64x4_epi64(_mm512_castph_si512(this->v), 1);
__m512i loSin = VectorBase<Len, Packing, _Float16>::sin_f32x16(_mm512_cvtph_ps(lo));
__m512i hiSin = VectorBase<Len, Packing, _Float16>::sin_f32x16(_mm512_cvtph_ps(hi));
__m256i lo_ph = _mm512_cvtps_ph(loSin, _MM_FROUND_TO_NEAREST_INT);
__m256i hi_ph = _mm512_cvtps_ph(hiSin, _MM_FROUND_TO_NEAREST_INT);
return VectorF16<Len, Packing>(_mm512_castsi512_ph(_mm512_inserti64x4(_mm512_castsi256_si512(lo_ph), hi_ph, 1)));
}
}
std::tuple<VectorF16<Len, Packing>, VectorF16<Len, Packing>> SinCos() {
if constexpr (std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
__m256 wide = _mm256_cvtph_ps(_mm_castph_si128(this->v));
__m256 s, c;
VectorBase<Len, Packing, _Float16>::sincos_f32x8(wide, s, c);
return {
VectorF16<Len, Packing>(_mm_castsi128_ph(_mm256_cvtps_ph(s, _MM_FROUND_TO_NEAREST_INT))),
VectorF16<Len, Packing>(_mm_castsi128_ph(_mm256_cvtps_ph(c, _MM_FROUND_TO_NEAREST_INT)))
};
} else if constexpr (std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
__m512 wide = _mm512_cvtph_ps(_mm256_castph_si256(this->v));
__m512 s, c;
VectorBase<Len, Packing, _Float16>::sincos_f32x16(wide, s, c);
return {
VectorF16<Len, Packing>(_mm256_castsi256_ph(_mm512_cvtps_ph(s, _MM_FROUND_TO_NEAREST_INT))),
VectorF16<Len, Packing>(_mm256_castsi256_ph(_mm512_cvtps_ph(c, _MM_FROUND_TO_NEAREST_INT)))
};
} else {
__m256i lo = _mm512_castsi512_si256(_mm512_castph_si512(this->v));
__m256i hi = _mm512_extracti64x4_epi64(_mm512_castph_si512(this->v), 1);
__m512 s_lo, c_lo, s_hi, c_hi;
VectorBase<Len, Packing, _Float16>::sincos_f32x16(_mm512_cvtph_ps(lo), s_lo, c_lo);
VectorBase<Len, Packing, _Float16>::sincos_f32x16(_mm512_cvtph_ps(hi), s_hi, c_hi);
auto pack = [](__m256i lo_ph, __m256i hi_ph) {
return _mm512_castsi512_ph(_mm512_inserti64x4(_mm512_castsi256_si512(lo_ph), hi_ph, 1));
};
return {
VectorF16<Len, Packing>(pack(_mm512_cvtps_ph(s_lo, _MM_FROUND_TO_NEAREST_INT), _mm512_cvtps_ph(s_hi, _MM_FROUND_TO_NEAREST_INT))),
VectorF16<Len, Packing>(pack( _mm512_cvtps_ph(c_lo, _MM_FROUND_TO_NEAREST_INT), _mm512_cvtps_ph(c_hi, _MM_FROUND_TO_NEAREST_INT)))
};
}
}
template <std::array<bool, Len> values>
constexpr VectorF16<Len, Packing> Negate() {
std::array<_Float16, VectorBase<Len, Packing, _Float16>::AlignmentElement> mask = VectorBase<Len, Packing, _Float16>::template GetNegateMask<values>();
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<Len, Packing>(_mm_castsi128_ph(_mm_xor_si128(_mm_castph_si128(this->v), _mm_loadu_epi16(mask.data()))));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<Len, Packing>(_mm256_castsi256_ph(_mm256_xor_si256(_mm256_castph_si256(this->v), _mm256_loadu_epi16(mask.data()))));
} else {
return VectorF16<Len, Packing>(_mm512_castsi512_ph(_mm512_xor_si512(_mm512_castph_si512(this->v), _mm512_loadu_epi16(mask.data()))));
}
}
static constexpr VectorF16<Len, Packing> MulitplyAdd(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b, VectorF16<Len, Packing> add) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<Len, Packing>(_mm_fmadd_ph(a.v, b.v, add.v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<Len, Packing>(_mm256_fmadd_ph(a.v, b.v, add.v));
} else {
return VectorF16<Len, Packing>(_mm512_fmadd_ph(a.v, b.v, add.v));
}
}
static constexpr VectorF16<Len, Packing> MulitplySub(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b, VectorF16<Len, Packing> sub) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<Len, Packing>(_mm_fmsub_ph(a.v, b.v, sub.v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<Len, Packing>(_mm256_fmsub_ph(a.v, b.v, sub.v));
} else {
return VectorF16<Len, Packing>(_mm512_fmsub_ph(a.v, b.v, sub.v));
}
}
constexpr static VectorF16<Len, Packing> Cross(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b) requires(Len == 3) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
constexpr std::array<std::uint8_t, VectorBase<Len, Packing, _Float16>::AlignmentElement*2> shuffleMask1 = VectorBase<Len, Packing, _Float16>::template GetShuffleMaskEpi8<{{1,2,0}}>();
__m128i shuffleVec1 = _mm_loadu_epi8(shuffleMask1.data());
__m128h row1 = _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(a.v), shuffleVec1));
__m128h row4 = _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(b.v), shuffleVec1));
constexpr std::array<std::uint8_t, VectorBase<Len, Packing, _Float16>::AlignmentElement*2> shuffleMask3 = VectorBase<Len, Packing, _Float16>::template GetShuffleMaskEpi8<{{2,0,1}}>();
__m128i shuffleVec3 = _mm_loadu_epi8(shuffleMask3.data());
__m128h row3 = _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(a.v), shuffleVec3));
__m128h row2 = _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(b.v), shuffleVec3));
__m128h result = _mm_mul_ph(row3, row4);
return _mm_fmsub_ph(row1,row2,result);
} else if constexpr (std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
constexpr std::array<std::uint8_t, VectorBase<Len, Packing, _Float16>::AlignmentElement*2> shuffleMask1 = VectorBase<Len, Packing, _Float16>::template GetShuffleMaskEpi8<{{1,2,0}}>();
__m512i shuffleVec1 = _mm512_castsi256_si512(_mm256_loadu_epi8(shuffleMask1.data()));
__m256h row1 = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(a.v)), shuffleVec1)));
__m256h row4 = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(b.v)), shuffleVec1)));
constexpr std::array<std::uint8_t, VectorBase<Len, Packing, _Float16>::AlignmentElement*2> shuffleMask3 = VectorBase<Len, Packing, _Float16>::template GetShuffleMaskEpi8<{{2,0,1}}>();
__m512i shuffleVec3 = _mm512_castsi256_si512(_mm256_loadu_epi8(shuffleMask3.data()));
__m256h row3 = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(a.v)), shuffleVec3)));
__m256h row2 = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(b.v)), shuffleVec3)));
__m256h result = _mm256_mul_ph(row3, row4);
return _mm256_fmsub_ph(row1,row2,result);
} else {
constexpr std::array<std::uint8_t, VectorBase<Len, Packing, _Float16>::AlignmentElement*2> shuffleMask1 = VectorBase<Len, Packing, _Float16>::template GetShuffleMaskEpi8<{{1,2,0}}>();
__m512i shuffleVec1 = _mm512_loadu_epi8(shuffleMask1.data());
__m512h row1 = _mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(a.v), shuffleVec1));
__m512h row4 = _mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(b.v), shuffleVec1));
constexpr std::array<std::uint8_t, VectorBase<Len, Packing, _Float16>::AlignmentElement*2> shuffleMask3 = VectorBase<Len, Packing, _Float16>::template GetShuffleMaskEpi8<{{2,0,1}}>();
__m512i shuffleVec3 = _mm512_loadu_epi8(shuffleMask3.data());
__m512h row3 = _mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(a.v), shuffleVec3));
__m512h row2 = _mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(b.v), shuffleVec3));
__m512h result = _mm512_mul_ph(row3, row4);
return _mm512_fmsub_ph(row1,row2,result);
}
}
constexpr static _Float16 Dot(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b) requires(Packing == 1) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
__m128h mul = _mm_mul_ph(a.v, b.v);
return _mm_reduce_add_ph(mul);
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
static_assert(std::is_same_v<decltype(a.v), typename VectorBase<Len, Packing, _Float16>::VectorType>, "a.v is NOT VectorBase<Len, Packing, _Float16>::VectorType");
__m256h mul = _mm256_mul_ph(a.v, b.v);
return _mm256_reduce_add_ph(mul);
} else {
__m512h mul = _mm512_mul_ph(a.v, b.v);
return _mm512_reduce_add_ph(mul);
}
}
template <const std::array<std::uint8_t, Len> ShuffleValues>
constexpr VectorF16<Len, Packing> Shuffle() {
if constexpr(VectorBase<Len, Packing, _Float16>::template CheckEpi32Shuffle<ShuffleValues>()) {
constexpr std::uint8_t imm = VectorBase<Len, Packing, _Float16>::template GetShuffleMaskEpi32<ShuffleValues>();
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<Len, Packing>(_mm_castsi128_ph(_mm_shuffle_epi32(_mm_castph_si128(this->v), imm)));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<Len, Packing>(_mm256_castsi256_ph(_mm256_shuffle_epi32(_mm256_castph_si256(this->v), imm)));
} else {
return VectorF16<Len, Packing>(_mm512_castsi512_ph(_mm512_shuffle_epi32(_mm512_castph_si512(this->v), imm)));
}
} else if constexpr(VectorBase<Len, Packing, _Float16>::template CheckEpi8Shuffle<ShuffleValues>()){
constexpr std::array<std::uint8_t, VectorBase<Len, Packing, _Float16>::Alignment> shuffleMask = VectorBase<Len, Packing, _Float16>::template GetShuffleMaskEpi8<ShuffleValues>();
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
__m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data());
return VectorF16<Len, Packing>(_mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(this->v), shuffleVec)));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
__m256i shuffleVec = _mm256_loadu_epi8(shuffleMask.data());
return VectorF16<Len, Packing>(_mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_shuffle_epi8(_mm512_castsi256_si512(_mm256_castph_si256(this->v)), _mm512_castsi256_si512(shuffleVec)))));
} else {
__m512i shuffleVec = _mm512_loadu_epi8(shuffleMask.data());
return VectorF16<Len, Packing>(_mm512_castsi512_ph(_mm512_shuffle_epi8(_mm512_castph_si512(this->v), shuffleVec)));
}
} else {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
constexpr std::array<std::uint8_t, VectorBase<Len, Packing, _Float16>::Alignment> shuffleMask = VectorBase<Len, Packing, _Float16>::template GetShuffleMaskEpi8<ShuffleValues>();
__m128i shuffleVec = _mm_loadu_epi8(shuffleMask.data());
return VectorF16<Len, Packing>(_mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(this->v), shuffleVec)));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
constexpr std::array<std::uint16_t, VectorBase<Len, Packing, _Float16>::AlignmentElement> permMask = VectorBase<Len, Packing, _Float16>::template GetPermuteMaskEpi16<ShuffleValues>();
__m256i permIdx = _mm256_loadu_epi16(permMask.data());
return VectorF16<Len, Packing>(_mm256_castsi256_ph(_mm256_permutexvar_epi16(permIdx, _mm256_castph_si256(this->v))));
} else {
constexpr std::array<std::uint16_t, VectorBase<Len, Packing, _Float16>::AlignmentElement> permMask = VectorBase<Len, Packing, _Float16>::template GetPermuteMaskEpi16<ShuffleValues>();
__m512i permIdx = _mm512_loadu_epi16(permMask.data());
return VectorF16<Len, Packing>(_mm512_castsi512_ph(_mm512_permutexvar_epi16(permIdx, _mm512_castph_si512(this->v))));
}
}
}
// Public variadic surface — one name per op, arity locked to BatchSize
// (or 2*BatchSize for Dot). Forwards to the *Pack helpers below which
// carry the SIMD bodies and per-(Len,Packing) requires clauses.
template <typename... Rest>
requires ((std::is_same_v<Rest, VectorF16<Len, Packing>> && ...) &&
(1 + sizeof...(Rest) == VectorBase<Len, Packing, _Float16>::BatchSize))
constexpr static auto Normalize(VectorF16<Len, Packing> first, Rest... rest) {
return NormalizePack(first, rest...);
}
template <typename... Rest>
requires ((std::is_same_v<Rest, VectorF16<Len, Packing>> && ...) &&
(1 + sizeof...(Rest) == VectorBase<Len, Packing, _Float16>::BatchSize))
constexpr static auto Length(VectorF16<Len, Packing> first, Rest... rest) {
return LengthPack(first, rest...);
}
template <typename... Rest>
requires ((std::is_same_v<Rest, VectorF16<Len, Packing>> && ...) &&
(1 + sizeof...(Rest) == VectorBase<Len, Packing, _Float16>::BatchSize))
constexpr static auto LengthSq(VectorF16<Len, Packing> first, Rest... rest) {
return LengthSqPack(first, rest...);
}
template <typename... Rest>
requires ((std::is_same_v<Rest, VectorF16<Len, Packing>> && ...) &&
(1 + sizeof...(Rest) == 2 * VectorBase<Len, Packing, _Float16>::BatchSize))
constexpr static auto Dot(VectorF16<Len, Packing> first, Rest... rest) {
return DotPack(first, rest...);
}
private:
constexpr static std::array<VectorF16<Len, Packing>, VectorBase<Len, Packing, _Float16>::BatchSize> NormalizePack(
VectorF16<Len, Packing> A,
VectorF16<Len, Packing> C,
VectorF16<Len, Packing> E,
VectorF16<Len, Packing> G
) requires(Len == 4 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
VectorF16<1, 8> lenght = LengthNoShuffle(A, E, C, G);
constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1};
__m128h one = _mm_loadu_ph(oneArr);
VectorF16<8, 1> fLenght(_mm_div_ph(one, lenght.v));
VectorF16<8, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,1,1,1,1}}>();
VectorF16<8, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,3,3,3,3}}>();
VectorF16<8, 1> fLenghtE = fLenght.template Shuffle<{{4,4,4,4,5,5,5,5}}>();
VectorF16<8, 1> fLenghtG = fLenght.template Shuffle<{{6,6,6,6,7,7,7,7}}>();
return {
_mm_mul_ph(A.v, fLenghtA.v),
_mm_mul_ph(C.v, fLenghtC.v),
_mm_mul_ph(E.v, fLenghtE.v),
_mm_mul_ph(G.v, fLenghtG.v)
};
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
VectorF16<1, 16> lenght = LengthNoShuffle(A, E, C, G);
constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
__m256h one = _mm256_loadu_ph(oneArr);
VectorF16<16, 1> fLenght(_mm256_div_ph(one, lenght.v));
VectorF16<16, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,1,1,1,1,8,8,8,8,9,9,9,9}}>();
VectorF16<16, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,3,3,3,3,10,10,10,10,11,11,11,11}}>();
VectorF16<16, 1> fLenghtE = fLenght.template Shuffle<{{4,4,4,4,5,5,5,5,12,12,12,12,13,13,13,13}}>();
VectorF16<16, 1> fLenghtG = fLenght.template Shuffle<{{6,6,6,6,7,7,7,7,14,14,14,14,15,15,15,15}}>();
return {
_mm256_mul_ph(A.v, fLenghtA.v),
_mm256_mul_ph(C.v, fLenghtC.v),
_mm256_mul_ph(E.v, fLenghtE.v),
_mm256_mul_ph(G.v, fLenghtG.v)
};
} else {
VectorF16<1, 32> lenght = LengthNoShuffle(A, E, C, G);
constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
__m512h one = _mm512_loadu_ph(oneArr);
VectorF16<32, 1> fLenght(_mm512_div_ph(one, lenght.v));
VectorF16<32, 1> fLenght2(lenght.v);
VectorF16<32, 1> fLenghtA = fLenght.template Shuffle<{{0,0,0,0,1,1,1,1,8,8,8,8,9,9,9,9,16,16,16,16,17,17,17,17,24,24,24,24,25,25,25,25}}>();
VectorF16<32, 1> fLenghtC = fLenght.template Shuffle<{{2,2,2,2,3,3,3,3,10,10,10,10,11,11,11,11,18,18,18,18,19,19,19,19,26,26,26,26,27,27,27,27}}>();
VectorF16<32, 1> fLenghtE = fLenght.template Shuffle<{{4,4,4,4,5,5,5,5,12,12,12,12,13,13,13,13,20,20,20,20,21,21,21,21,28,28,28,28,29,29,29,29}}>();
VectorF16<32, 1> fLenghtG = fLenght.template Shuffle<{{6,6,6,6,7,7,7,7,14,14,14,14,15,15,15,15,22,22,22,22,23,23,23,23,30,30,30,30,31,31,31,31}}>();
return {
VectorF16<Len, Packing>(_mm512_mul_ph(A.v, fLenghtA.v)),
VectorF16<Len, Packing>(_mm512_mul_ph(C.v, fLenghtC.v)),
VectorF16<Len, Packing>(_mm512_mul_ph(E.v, fLenghtE.v)),
VectorF16<Len, Packing>(_mm512_mul_ph(G.v, fLenghtG.v)),
};
}
}
constexpr static std::array<VectorF16<Len, Packing>, VectorBase<Len, Packing, _Float16>::BatchSize> NormalizePack(
VectorF16<Len, Packing> A,
VectorF16<Len, Packing> E
) requires(Len == 2 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
VectorF16<1, 8> lenght = LengthNoShuffle(A, E);
constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1};
__m128h one = _mm_loadu_ph(oneArr);
VectorF16<8, 1> fLenght(_mm_div_ph(one, lenght.v));
VectorF16<8, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,2,2,3,3}}>();
VectorF16<8, 1> fLenghtE = fLenght.template Shuffle<{{4,4,5,5,6,6,7,7}}>();
return {
_mm_mul_ph(A.v, fLenghtA.v),
_mm_mul_ph(E.v, fLenghtE.v),
};
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
VectorF16<1, 16> lenght = LengthNoShuffle(A, E);
constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
__m256h one = _mm256_loadu_ph(oneArr);
VectorF16<16, 1> fLenght(_mm256_div_ph(one, lenght.v));
VectorF16<16, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11}}>();
VectorF16<16, 1> fLenghtE = fLenght.template Shuffle<{{4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15}}>();
return {
_mm256_mul_ph(A.v, fLenghtA.v),
_mm256_mul_ph(E.v, fLenghtE.v),
};
} else {
VectorF16<1, 32> lenght = LengthNoShuffle(A, E);
constexpr _Float16 oneArr[] {1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1};
__m512h one = _mm512_loadu_ph(oneArr);
VectorF16<32, 1> fLenght(_mm512_div_ph(one, lenght.v));
VectorF16<32, 1> fLenghtA = fLenght.template Shuffle<{{0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11,16,16,17,17,18,18,19,19,24,24,25,25,26,26,27,27}}>();
VectorF16<32, 1> fLenghtE = fLenght.template Shuffle<{{4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15,20,20,21,21,22,22,23,23,28,28,29,29,30,30,31,31}}>();
return {
_mm512_mul_ph(A.v, fLenghtA.v),
_mm512_mul_ph(E.v, fLenghtE.v),
};
}
}
constexpr static VectorF16<1, Packing*4> LengthPack(
VectorF16<Len, Packing> A,
VectorF16<Len, Packing> C,
VectorF16<Len, Packing> E,
VectorF16<Len, Packing> G
) requires(Len == 4 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
VectorF16<1, Packing*4> lenghtSq = LengthSqPack(A, C, E, G);
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<1, Packing*4>(_mm_sqrt_ph(lenghtSq.v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<1, Packing*4>(_mm256_sqrt_ph(lenghtSq.v));
} else {
return VectorF16<1, Packing*4>(_mm512_sqrt_ph(lenghtSq.v));
}
}
constexpr static VectorF16<1, Packing*2> LengthPack(
VectorF16<Len, Packing> A,
VectorF16<Len, Packing> E
) requires(Len == 2 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
VectorF16<1, Packing*2> lenghtSq = LengthSqPack(A, E);
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<1, Packing*2>(_mm_sqrt_ph(lenghtSq.v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<1, Packing*2>(_mm256_sqrt_ph(lenghtSq.v));
} else {
return VectorF16<1, Packing*2>(_mm512_sqrt_ph(lenghtSq.v));
}
}
constexpr static VectorF16<1, Packing*4> LengthSqPack(
VectorF16<Len, Packing> A,
VectorF16<Len, Packing> C,
VectorF16<Len, Packing> E,
VectorF16<Len, Packing> G
) requires(Len == 4 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
return DotPack(A, A, C, C, E, E, G, G);
}
constexpr static VectorF16<1, Packing*2> LengthSqPack(
VectorF16<Len, Packing> A,
VectorF16<Len, Packing> E
) requires(Len == 2 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
return DotPack(A, A, E, E);
}
constexpr static VectorF16<1, Packing*4> DotPack(
VectorF16<Len, Packing> A0, VectorF16<Len, Packing> A1,
VectorF16<Len, Packing> C0, VectorF16<Len, Packing> C1,
VectorF16<Len, Packing> E0, VectorF16<Len, Packing> E1,
VectorF16<Len, Packing> G0, VectorF16<Len, Packing> G1
) requires(Len == 4 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return DotNoShuffle(A0, A1, E0, E1, C0, C1, G0, G1);
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
VectorF16<16, 1> vec(DotNoShuffle(A0, A1, C0, C1, E0, E1, G0, G1).v);
vec = vec.template Shuffle<{{
0,1,8,9,
4,5,12,13,
2,3,10,11,
6,7,14,15
}}>();
return vec.v;
} else {
VectorF16<32, 1> vec(DotNoShuffle(A0, A1, C0, C1, E0, E1, G0, G1).v);
vec = vec.template Shuffle<{{
0,1,8,9,
16,17,24,25,
4,5,12,13,
20,21,28,29,
2,3,10,11,
18,19,24,25,
6,7,14,15,
22,23,30,31
}}>();
return vec.v;
}
}
constexpr static VectorF16<1, Packing*2> DotPack(
VectorF16<Len, Packing> A0, VectorF16<Len, Packing> A1,
VectorF16<Len, Packing> E0, VectorF16<Len, Packing> E1
) requires(Len == 2 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return DotNoShuffle(A0, A1, E0, E1);
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
VectorF16<16, 1> vec(DotNoShuffle(A0, A1, E0, E1).v);
vec = vec.template Shuffle<{{0,1,2,3,8,9,10,11,4,5,6,7,12,13,14,15}}>();
return vec.v;
} else {
VectorF16<32, 1> vec(DotNoShuffle(A0, A1, E0, E1).v);
vec = vec.template Shuffle<{{0,1,2,3,8,9,10,11,16,17,18,19,24,25,26,27,4,5,6,7,12,13,14,15,20,21,22,23,28,29,30,31}}>();
return vec.v;
}
}
private:
constexpr static VectorF16<1, Packing*4> LengthNoShuffle(
VectorF16<Len, Packing> A,
VectorF16<Len, Packing> C,
VectorF16<Len, Packing> E,
VectorF16<Len, Packing> G
) requires(Len == 4 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
VectorF16<1, Packing*4> lenghtSq = LengthSqNoShuffle(A, C, E, G);
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<1, Packing*4>(_mm_sqrt_ph(lenghtSq.v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<1, Packing*4>(_mm256_sqrt_ph(lenghtSq.v));
} else {
return VectorF16<1, Packing*4>(_mm512_sqrt_ph(lenghtSq.v));
}
}
constexpr static VectorF16<1, Packing*2> LengthNoShuffle(
VectorF16<Len, Packing> A,
VectorF16<Len, Packing> E
) requires(Len == 2 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
VectorF16<1, Packing*2> lenghtSq = LengthSqNoShuffle(A, E);
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return VectorF16<1, Packing*2>(_mm_sqrt_ph(lenghtSq.v));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
return VectorF16<1, Packing*2>(_mm256_sqrt_ph(lenghtSq.v));
} else {
return VectorF16<1, Packing*2>(_mm512_sqrt_ph(lenghtSq.v));
}
}
constexpr static VectorF16<1, Packing*4> LengthSqNoShuffle(
VectorF16<Len, Packing> A,
VectorF16<Len, Packing> C,
VectorF16<Len, Packing> E,
VectorF16<Len, Packing> G
) requires(Len == 4 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
return DotNoShuffle(A, A, C, C, E, E, G, G);
}
constexpr static VectorF16<1, Packing*2> LengthSqNoShuffle(
VectorF16<Len, Packing> A,
VectorF16<Len, Packing> E
) requires(Len == 2 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
return DotNoShuffle(A, A, E, E);
}
constexpr static VectorF16<1, Packing*4> DotNoShuffle(
VectorF16<Len, Packing> A0, VectorF16<Len, Packing> A1,
VectorF16<Len, Packing> C0, VectorF16<Len, Packing> C1,
VectorF16<Len, Packing> E0, VectorF16<Len, Packing> E1,
VectorF16<Len, Packing> G0, VectorF16<Len, Packing> G1
) requires(Len == 4 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
__m128h mulA = _mm_mul_ph(A0.v, A1.v);
__m128h mulC = _mm_mul_ph(C0.v, C1.v);
__m128i row12Temp1 = _mm_unpacklo_epi16(_mm_castph_si128(mulA), _mm_castph_si128(mulC)); // A1 C1 A2 C2 A3 C3 A4 C4
__m128i row34Temp1 = _mm_unpackhi_epi16(_mm_castph_si128(mulA), _mm_castph_si128(mulC)); // B1 D1 B2 D2 B3 D3 B4 D4
__m128i row1TempTemp1 = row12Temp1;
__m128h mulE = _mm_mul_ph(E0.v, E1.v);
__m128h mulG = _mm_mul_ph(G0.v, G1.v);
__m128i row12Temp2 = _mm_unpacklo_epi16(_mm_castph_si128(mulE), _mm_castph_si128(mulG)); // E1 G1 E2 G2 E3 G3 E4 G4
__m128i row12Temp2Temp = row12Temp2;
__m128i row34Temp2 = _mm_unpackhi_epi16(_mm_castph_si128(mulE), _mm_castph_si128(mulG)); // F1 H1 F2 H2 F3 H3 F4 H4
row12Temp1 = _mm_unpacklo_epi16(row12Temp1, row12Temp2); // A1 E1 C1 G1 A2 E2 C2 G2
row12Temp2 = _mm_unpacklo_epi16(row34Temp1, row34Temp2); // B1 F1 D1 H1 B2 F2 D2 H2
row34Temp2 = _mm_unpackhi_epi16(row34Temp1, row34Temp2); // B3 F3 D3 H3 B4 F4 D4 H4
row34Temp1 = _mm_unpackhi_epi16(row1TempTemp1, row12Temp2Temp); // A3 E3 C3 G3 A4 E4 C4 G4
__m128h row1 = _mm_castsi128_ph(_mm_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 E1 F1 C1 D1 G1 H1
__m128h row2 = _mm_castsi128_ph(_mm_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 E2 F2 C2 D2 G2 H2
__m128h row3 = _mm_castsi128_ph(_mm_unpacklo_epi16(row34Temp1, row34Temp2));// A3 B3 E3 F3 C3 D3 G3 H3
__m128h row4 = _mm_castsi128_ph(_mm_unpackhi_epi16(row34Temp1, row34Temp2));// A4 B4 E4 F4 C4 D4 G4 H4
row1 = _mm_add_ph(row1, row2);
row1 = _mm_add_ph(row1, row3);
row1 = _mm_add_ph(row1, row4);
return row1;
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
__m256h mulA = _mm256_mul_ph(A0.v, A1.v);
__m256h mulC = _mm256_mul_ph(C0.v, C1.v);
__m256i row12Temp1 = _mm256_unpacklo_epi16(_mm256_castph_si256(mulA), _mm256_castph_si256(mulC)); // A1 C1 A2 C2 A3 C3 A4 C4
__m256i row34Temp1 = _mm256_unpackhi_epi16(_mm256_castph_si256(mulA), _mm256_castph_si256(mulC)); // B1 D1 B2 D2 B3 D3 B4 D4
__m256i row1TempTemp1 = row12Temp1;
__m256h mulE = _mm256_mul_ph(E0.v, E1.v);
__m256h mulG = _mm256_mul_ph(G0.v, G1.v);
__m256i row12Temp2 = _mm256_unpacklo_epi16(_mm256_castph_si256(mulE), _mm256_castph_si256(mulG)); // E1 G1 E2 G2 E3 G3 E4 G4
__m256i row12Temp2Temp = row12Temp2;
__m256i row34Temp2 = _mm256_unpackhi_epi16(_mm256_castph_si256(mulE), _mm256_castph_si256(mulG)); // F1 H1 F2 H2 F3 H3 F4 H4
row12Temp1 = _mm256_unpacklo_epi16(row12Temp1, row12Temp2); // A1 E1 C1 G1 A2 E2 C2 G2
row12Temp2 = _mm256_unpacklo_epi16(row34Temp1, row34Temp2); // B1 F1 D1 H1 B2 F2 D2 H2
row34Temp2 = _mm256_unpackhi_epi16(row34Temp1, row34Temp2); // B3 F3 D3 H3 B4 F4 D4 H4
row34Temp1 = _mm256_unpackhi_epi16(row1TempTemp1, row12Temp2Temp); // A3 E3 C3 G3 A4 E4 C4 G4
__m256h row1 = _mm256_castsi256_ph(_mm256_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 E1 F1 C1 D1 G1 H1
__m256h row2 = _mm256_castsi256_ph(_mm256_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 E2 F2 C2 D2 G2 H2
__m256h row3 = _mm256_castsi256_ph(_mm256_unpacklo_epi16(row34Temp1, row34Temp2));// A3 B3 E3 F3 C3 D3 G3 H3
__m256h row4 = _mm256_castsi256_ph(_mm256_unpackhi_epi16(row34Temp1, row34Temp2));// A4 B4 E4 F4 C4 D4 G4 H4
row1 = _mm256_add_ph(row1, row2);
row1 = _mm256_add_ph(row1, row3);
row1 = _mm256_add_ph(row1, row4);
return row1;
} else {
__m512h mulA = _mm512_mul_ph(A0.v, A1.v);
__m512h mulC = _mm512_mul_ph(C0.v, C1.v);
__m512i row12Temp1 = _mm512_unpacklo_epi16(_mm512_castph_si512(mulA), _mm512_castph_si512(mulC)); // A1 C1 A2 C2 A3 C3 A4 C4
__m512i row34Temp1 = _mm512_unpackhi_epi16(_mm512_castph_si512(mulA), _mm512_castph_si512(mulC)); // B1 D1 B2 D2 B3 D3 B4 D4
__m512i row1TempTemp1 = row12Temp1;
__m512h mulE = _mm512_mul_ph(E0.v, E1.v);
__m512h mulG = _mm512_mul_ph(G0.v, G1.v);
__m512i row12Temp2 = _mm512_unpacklo_epi16(_mm512_castph_si512(mulE), _mm512_castph_si512(mulG)); // E1 G1 E2 G2 E3 G3 E4 G4
__m512i row12Temp2Temp = row12Temp2;
__m512i row34Temp2 = _mm512_unpackhi_epi16(_mm512_castph_si512(mulE), _mm512_castph_si512(mulG)); // F1 H1 F2 H2 F3 H3 F4 H4
row12Temp1 = _mm512_unpacklo_epi16(row12Temp1, row12Temp2); // A1 E1 C1 G1 A2 E2 C2 G2
row12Temp2 = _mm512_unpacklo_epi16(row34Temp1, row34Temp2); // B1 F1 D1 H1 B2 F2 D2 H2
row34Temp2 = _mm512_unpackhi_epi16(row34Temp1, row34Temp2); // B3 F3 D3 H3 B4 F4 D4 H4
row34Temp1 = _mm512_unpackhi_epi16(row1TempTemp1, row12Temp2Temp); // A3 E3 C3 G3 A4 E4 C4 G4
__m512h row1 = _mm512_castsi512_ph(_mm512_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 E1 F1 C1 D1 G1 H1
__m512h row2 = _mm512_castsi512_ph(_mm512_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 E2 F2 C2 D2 G2 H2
__m512h row3 = _mm512_castsi512_ph(_mm512_unpacklo_epi16(row34Temp1, row34Temp2));// A3 B3 E3 F3 C3 D3 G3 H3
__m512h row4 = _mm512_castsi512_ph(_mm512_unpackhi_epi16(row34Temp1, row34Temp2));// A4 B4 E4 F4 C4 D4 G4 H4
row1 = _mm512_add_ph(row1, row2);
row1 = _mm512_add_ph(row1, row3);
row1 = _mm512_add_ph(row1, row4);
return row1;
}
}
constexpr static VectorF16<1, Packing*2> DotNoShuffle(
VectorF16<Len, Packing> A0, VectorF16<Len, Packing> A1,
VectorF16<Len, Packing> E0, VectorF16<Len, Packing> E1
) requires(Len == 2 && Packing*Len == VectorBase<Len, Packing, _Float16>::AlignmentElement) {
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
__m128h mulA = _mm_mul_ph(A0.v, A1.v);
__m128h mulE = _mm_mul_ph(E0.v, E1.v);
__m128i row12Temp1 = _mm_unpacklo_epi16(_mm_castph_si128(mulA), _mm_castph_si128(mulE)); // A1 E1 A2 E2 B1 F1 B2 F2
__m128i row12Temp2 = _mm_unpackhi_epi16(_mm_castph_si128(mulA), _mm_castph_si128(mulE)); // C1 G1 C2 G2 D1 H1 D2 H2
__m128i row12Temp1Temp = row12Temp1;
row12Temp1 = _mm_unpacklo_epi16(row12Temp1, row12Temp2); // A1 C1 E1 G1 A2 C2 E2 G2
row12Temp2 = _mm_unpackhi_epi16(row12Temp1Temp, row12Temp2); // B1 D1 F1 H1 B2 D2 F2 H2
__m128h row1 = _mm_castsi128_ph(_mm_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 C1 D1 E1 F1 G1 H1
__m128h row2 = _mm_castsi128_ph(_mm_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 C2 D2 E2 F2 G2 H2
return _mm_add_ph(row1, row2);
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
__m256h mulA = _mm256_mul_ph(A0.v, A1.v);
__m256h mulE = _mm256_mul_ph(E0.v, E1.v);
__m256i row12Temp1 = _mm256_unpacklo_epi16(_mm256_castph_si256(mulA), _mm256_castph_si256(mulE)); // A1 E1 A2 E2 B1 F1 B2 F2
__m256i row12Temp2 = _mm256_unpackhi_epi16(_mm256_castph_si256(mulA), _mm256_castph_si256(mulE)); // C1 G1 C2 G2 D1 H1 D2 H2
__m256i row12Temp1Temp = row12Temp1;
row12Temp1 = _mm256_unpacklo_epi16(row12Temp1, row12Temp2); // A1 C1 E1 G1 A2 C2 E2 G2
row12Temp2 = _mm256_unpackhi_epi16(row12Temp1Temp, row12Temp2); // B1 D1 F1 H1 B2 D2 F2 H2
__m256h row1 = _mm256_castsi256_ph(_mm256_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 C1 D1 E1 F1 G1 H1
__m256h row2 = _mm256_castsi256_ph(_mm256_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 C2 D2 E2 F2 G2 H2
__m256h result = _mm256_add_ph(row1, row2);
return result;
} else {
__m512h mulA = _mm512_mul_ph(A0.v, A1.v);
__m512h mulE = _mm512_mul_ph(E0.v, E1.v);
__m512i row12Temp1 = _mm512_unpacklo_epi16(_mm512_castph_si512(mulA), _mm512_castph_si512(mulE)); // A1 E1 A2 E2 B1 F1 B2 F2
__m512i row12Temp2 = _mm512_unpackhi_epi16(_mm512_castph_si512(mulA), _mm512_castph_si512(mulE)); // C1 G1 C2 G2 D1 H1 D2 H2
__m512i row12Temp1Temp = row12Temp1;
row12Temp1 = _mm512_unpacklo_epi16(row12Temp1, row12Temp2); // A1 C1 E1 G1 A2 C2 E2 G2
row12Temp2 = _mm512_unpackhi_epi16(row12Temp1Temp, row12Temp2); // B1 D1 F1 H1 B2 D2 F2 H2
__m512h row1 = _mm512_castsi512_ph(_mm512_unpacklo_epi16(row12Temp1, row12Temp2));// A1 B1 C1 D1 E1 F1 G1 H1
__m512h row2 = _mm512_castsi512_ph(_mm512_unpackhi_epi16(row12Temp1, row12Temp2));// A2 B2 C2 D2 E2 F2 G2 H2
__m512h result = _mm512_add_ph(row1, row2);
return result;
}
}
public:
template <std::array<bool, Len> ShuffleValues>
constexpr static VectorF16<Len, Packing> Blend(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b) {
constexpr auto mask = VectorBase<Len, Packing, _Float16>::template GetBlendMaskEpi16<ShuffleValues>();
if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m128h>) {
return _mm_castsi128_ph(_mm_blend_epi16(_mm_castph_si128(a.v), _mm_castph_si128(b.v), mask));
} else if constexpr(std::is_same_v<typename VectorBase<Len, Packing, _Float16>::VectorType, __m256h>) {
#ifndef __AVX512BW__
#ifndef __AVX512VL__
static_assert(false, "No __AVX512BW__ and __AVX512VL__ support");
#endif
#endif
return _mm256_castsi256_ph(_mm256_mask_blend_epi16(mask, _mm256_castph_si256(a.v), _mm256_castph_si256(b.v)));
} else {
return _mm512_castsi512_ph(_mm512_mask_blend_epi16(mask, _mm512_castph_si512(a.v), _mm512_castph_si512(b.v)));
}
}
constexpr static VectorF16<Len, Packing> Rotate(VectorF16<3, Packing> v, VectorF16<4, Packing> q) requires(Len == 3) {
VectorF16<3, Packing> qv(q);
VectorF16<Len, Packing> t = Cross(qv, v) * _Float16(2);
return v + t * q.template Shuffle<{{3,3,3,3}}>() + Cross(qv, t);
}
constexpr static VectorF16<4, 2> RotatePivot(VectorF16<3, Packing> v, VectorF16<4, Packing> q, VectorF16<3, Packing> pivot) requires(Len == 3) {
VectorF16<Len, Packing> translated = v - pivot;
VectorF16<3, Packing> qv(q.v);
VectorF16<Len, Packing> t = Cross(qv, translated) * _Float16(2);
VectorF16<Len, Packing> rotated = translated + t * q.template Shuffle<{{3,3,3,3}}>() + Cross(qv, t);
return rotated + pivot;
}
constexpr static VectorF16<4, Packing> QuanternionFromEuler(VectorF16<3, Packing> EulerHalf) requires(Len == 4) {
std::tuple<VectorF16<3, Packing>, VectorF16<3, Packing>> sinCos = EulerHalf.SinCos();
VectorF16<4, Packing> sin = std::get<0>(sinCos);
VectorF16<4, Packing> cos = std::get<1>(sinCos);
VectorF16<4, Packing> row1 = cos.template Shuffle<{{0,0,0,0}}>();
row1 = Blend<{{0,1,1,1}}>(sin, row1);
VectorF16<4, Packing> row2 = cos.template Shuffle<{{1,1,1,1}}>();
row2 = Blend<{{1,0,1,1}}>(sin, row2);
row1 *= row2;
VectorF16<4, Packing> row3 = cos.template Shuffle<{{2,2,2,2}}>();
row3 = Blend<{{1,1,0,1}}>(sin, row3);
row1 *= row3;
VectorF16<4, Packing> row4 = sin.template Shuffle<{{0,0,0,0}}>();
row4 = Blend<{{0,1,1,1}}>(cos, row4);
VectorF16<4, Packing> row5 = sin.template Shuffle<{{1,1,1,1}}>();
row5 = Blend<{{1,0,1,1}}>(cos, row5);
row4 *= row5;
VectorF16<4, Packing> row6 = sin.template Shuffle<{{2,2,2,2}}>();
row6 = Blend<{{1,1,0,1}}>(cos, row6);
row6 = row6.template Negate<{{true,false,true,false}}>();
row1 = MulitplyAdd(row4, row6, row1);
return row1;
}
};
}
#elif defined(__aarch64__) && defined(__ARM_NEON) && defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC)
// AArch64 NEON with ARMv8.2-FP16 half-precision arithmetic. Built the same
// way as the NEON VectorF32: every vector is one 128-bit register, here a
// float16x8_t with eight lanes, and shuffles and blends are compile-time lane
// maps handed to __builtin_shufflevector.
namespace Crafter {
export template <std::uint8_t Len, std::uint8_t Packing>
struct VectorF16 : public VectorBase<Len, Packing, _Float16> {
template <std::uint8_t Len2, std::uint8_t Packing2>
friend struct VectorF16;
using Base = VectorBase<Len, Packing, _Float16>;
static constexpr std::uint8_t NElems = Base::AlignmentElement;
static_assert(NElems == 8, "NEON VectorF16 assumes 8-lane vectors");
private:
// Source lane for each of the eight output lanes: 0..7 pick from the
// first operand of Permute, 8..15 from the second. Every map names a
// real source lane for all eight outputs, padding included, so no
// lane of a result is ever undefined.
using LaneMap = std::array<int, 8>;
static constexpr LaneMap InPlace{0, 1, 2, 3, 4, 5, 6, 7};
template <LaneMap Lanes, typename Register>
static constexpr Register Permute(Register a, Register b) {
return __builtin_shufflevector(a, b, Lanes[0], Lanes[1], Lanes[2], Lanes[3], Lanes[4], Lanes[5], Lanes[6], Lanes[7]);
}
// Per-pack shuffle; padding lanes stay where they are.
template <std::array<std::uint8_t, Len> ShuffleValues>
static consteval LaneMap GetShuffleLanes() {
LaneMap lanes = InPlace;
for (std::uint8_t p = 0; p < Packing; ++p)
for (std::uint8_t i = 0; i < Len; ++i)
lanes[p * Len + i] = p * Len + ShuffleValues[i];
return lanes;
}
// Per-pack blend: the second operand where the mask is set, the first
// elsewhere.
template <std::array<bool, Len> BlendValues>
static consteval LaneMap GetBlendLanes() {
LaneMap lanes = InPlace;
for (std::uint8_t p = 0; p < Packing; ++p)
for (std::uint8_t i = 0; i < Len; ++i)
if (BlendValues[i]) lanes[p * Len + i] += 8;
return lanes;
}
// Moves every sub-vector to its slot in a (ToLen, ToPacking) layout,
// carrying over the components both layouts have. The remaining lanes
// are left as they were.
template <std::uint8_t ToLen, std::uint8_t ToPacking>
static consteval LaneMap GetResizeLanes() {
LaneMap lanes = InPlace;
for (std::uint8_t p = 0; p < std::min(Packing, ToPacking); ++p)
for (std::uint8_t i = 0; i < std::min(Len, ToLen); ++i)
lanes[p * ToLen + i] = p * Len + i;
return lanes;
}
// Normalize keeps one scale per sub-vector in a VectorF16<1, N*Packing>;
// this spreads the scales of input `Index` over the Len lanes of each
// of its sub-vectors. Padding lanes repeat the last scale.
template <std::uint8_t Index>
static consteval LaneMap GetSpreadLanes() {
LaneMap lanes{};
for (std::uint8_t i = 0; i < 8; ++i)
lanes[i] = Index * Packing + std::min(i / Len, Packing - 1);
return lanes;
}
// QuanternionFromEuler factor row out of the sin (first operand) and
// cos (second operand) of a VectorF16<3, Packing>: in each quaternion,
// lane `Axis` takes the sin of that axis and the other three lanes
// its cos, or the other way around when `Swap` is set.
template <std::uint8_t Axis, bool Swap>
static consteval LaneMap GetEulerLanes() {
LaneMap lanes = InPlace;
for (std::uint8_t p = 0; p < Packing; ++p)
for (std::uint8_t i = 0; i < 4; ++i)
lanes[p * 4 + i] = ((i == Axis) != Swap ? 0 : 8) + p * 3 + Axis;
return lanes;
}
// The batched reductions sum sub-vectors with faddp, which adds the
// adjacent lane pairs of two registers at once. That takes every
// sub-vector in a power-of-two slot of lanes: `Slot` lanes wide,
// `Slots` of them per register.
static constexpr std::uint8_t Slot = std::bit_ceil(Len);
static constexpr std::uint8_t Slots = 8 / Slot;
// Spreads the sub-vectors out to one slot each and zero-fills the rest
// (lane 8 is a zero from Permute's second operand). Power-of-two Len
// sits in its slots already, and whatever shares the register with it
// ends up in sums of its own.
static consteval LaneMap GetSlotLanes() {
LaneMap lanes{};
for (std::uint8_t i = 0; i < 8; ++i)
lanes[i] = (i / Slot < Packing && i % Slot < Len) ? (i / Slot) * Len + i % Slot : 8;
return lanes;
}
static constexpr float16x8_t ToSlots(float16x8_t m) {
if constexpr (Slot != Len) {
return Permute<GetSlotLanes()>(m, vdupq_n_f16(0));
} else {
return m;
}
}
// Once the faddp levels are done, a register holds the sums of `Slot`
// inputs, each in the leading Packing lanes of a group of Slots. This
// is the lane the i-th of those sums sits in.
static consteval int SumLane(std::uint8_t i) {
return (i / Packing) * Slots + i % Packing;
}
static consteval LaneMap GetSumLanes() {
LaneMap lanes = InPlace;
for (std::uint8_t i = 0; i < Slot * Packing; ++i) lanes[i] = SumLane(i);
return lanes;
}
// Puts the first `Width` values of a, then the first `Width` of b, at
// the front. `Sums` says the values still sit where the faddp levels
// left them (see SumLane) instead of already being at the front.
template <std::uint8_t Width, bool Sums>
static consteval LaneMap GetJoinLanes() {
LaneMap lanes = InPlace;
for (std::uint8_t i = 0; i < Width; ++i) {
lanes[i] = Sums ? SumLane(i) : i;
lanes[Width + i] = 8 + lanes[i];
}
return lanes;
}
// Concatenates the sums of the first `Count` registers, `Width` of
// them per register, halving the register count per step.
template <std::uint8_t Count, std::uint8_t Width, bool Sums>
static constexpr float16x8_t Join(std::array<float16x8_t, Base::BatchSize> r) {
if constexpr (Count > 1) {
for (std::uint8_t i = 0; i < Count / 2; ++i)
r[i] = Permute<GetJoinLanes<Width, Sums>()>(r[2 * i], r[2 * i + 1]);
return Join<Count / 2, 2 * Width, false>(r);
} else if constexpr (Sums) {
return Permute<GetSumLanes()>(r[0], r[0]);
} else {
return r[0];
}
}
// Collapses every sub-vector of the BatchSize registers into the sum
// of its components: result lane i*Packing+p is the sum of lanes
// [p*Len, p*Len+Len) of m[i]. Each faddp level halves the slot width
// and packs two registers into one, so log2(Slot) levels leave plain
// sums that Join then lines up.
static constexpr float16x8_t SumPacks(std::array<float16x8_t, Base::BatchSize> m) {
constexpr std::uint8_t N = Base::BatchSize;
static_assert(N % Slot == 0);
for (float16x8_t& product : m) product = ToSlots(product);
std::uint8_t count = N;
for (std::uint8_t width = Slot; width > 1; width /= 2) {
count /= 2;
for (std::uint8_t i = 0; i < count; ++i) m[i] = vpaddq_f16(m[2 * i], m[2 * i + 1]);
}
return Join<N / Slot, Slot * Packing, true>(m);
}
public:
constexpr VectorF16() = default;
constexpr VectorF16(float16x8_t vv) { this->v = vv; }
constexpr VectorF16(const _Float16* vB) { Load(vB); }
constexpr VectorF16(_Float16 val) { this->v = vdupq_n_f16(val); }
// Through memcpy because arm_neon.h types its fp16 pointers as __fp16,
// which _Float16 may not alias.
constexpr void Load(const _Float16* vB) { std::memcpy(&this->v, vB, sizeof(this->v)); }
constexpr void Store(_Float16* vB) const { std::memcpy(vB, &this->v, sizeof(this->v)); }
template<typename T>
constexpr std::array<_Float16, NElems> Store() const {
std::array<_Float16, NElems> r;
Store(r.data());
return r;
}
template <std::uint8_t BLen, std::uint8_t BPacking>
constexpr operator VectorF16<BLen, BPacking>() const {
return VectorF16<BLen, BPacking>(Permute<GetResizeLanes<BLen, BPacking>()>(this->v, this->v));
}
constexpr VectorF16<Len, Packing> operator+(VectorF16<Len, Packing> b) const { return VectorF16<Len, Packing>(vaddq_f16(this->v, b.v)); }
constexpr VectorF16<Len, Packing> operator-(VectorF16<Len, Packing> b) const { return VectorF16<Len, Packing>(vsubq_f16(this->v, b.v)); }
constexpr VectorF16<Len, Packing> operator*(VectorF16<Len, Packing> b) const { return VectorF16<Len, Packing>(vmulq_f16(this->v, b.v)); }
constexpr VectorF16<Len, Packing> operator/(VectorF16<Len, Packing> b) const { return VectorF16<Len, Packing>(vdivq_f16(this->v, b.v)); }
constexpr void operator+=(VectorF16<Len, Packing> b) { this->v = vaddq_f16(this->v, b.v); }
constexpr void operator-=(VectorF16<Len, Packing> b) { this->v = vsubq_f16(this->v, b.v); }
constexpr void operator*=(VectorF16<Len, Packing> b) { this->v = vmulq_f16(this->v, b.v); }
constexpr void operator/=(VectorF16<Len, Packing> b) { this->v = vdivq_f16(this->v, b.v); }
constexpr VectorF16<Len, Packing> operator+(_Float16 b) const { return *this + VectorF16<Len, Packing>(b); }
constexpr VectorF16<Len, Packing> operator-(_Float16 b) const { return *this - VectorF16<Len, Packing>(b); }
constexpr VectorF16<Len, Packing> operator*(_Float16 b) const { return *this * VectorF16<Len, Packing>(b); }
constexpr VectorF16<Len, Packing> operator/(_Float16 b) const { return *this / VectorF16<Len, Packing>(b); }
constexpr void operator+=(_Float16 b) { *this += VectorF16<Len, Packing>(b); }
constexpr void operator-=(_Float16 b) { *this -= VectorF16<Len, Packing>(b); }
constexpr void operator*=(_Float16 b) { *this *= VectorF16<Len, Packing>(b); }
constexpr void operator/=(_Float16 b) { *this /= VectorF16<Len, Packing>(b); }
constexpr VectorF16<Len, Packing> operator-() const { return VectorF16<Len, Packing>(vnegq_f16(this->v)); }
// Only the Len*Packing lanes in use take part; padding lanes can hold
// anything. Narrowing the compare result to a byte per lane puts all
// eight in one 64-bit integer, where the lanes in use are its low
// bytes.
constexpr bool operator==(VectorF16<Len, Packing> b) const {
constexpr std::uint64_t used = ~0ull >> (64 - 8 * Len * Packing);
std::uint64_t equal = vget_lane_u64(vreinterpret_u64_u8(vmovn_u16(vceqq_f16(this->v, b.v))), 0);
return (equal & used) == used;
}
constexpr bool operator!=(VectorF16<Len, Packing> b) const { return !(*this == b); }
template<std::uint32_t ExtractLen>
constexpr VectorF16<ExtractLen, Packing> ExtractLo() const {
return VectorF16<ExtractLen, Packing>(Permute<GetResizeLanes<ExtractLen, Packing>()>(this->v, this->v));
}
// Sin and cos run in single precision: the register widens to fp32,
// goes through the shared kernels and narrows back. The upper half is
// skipped when it is all padding.
constexpr std::tuple<VectorF16<Len, Packing>, VectorF16<Len, Packing>> SinCos() const {
float32x4_t sinLo, cosLo;
Base::sincos_f32x4(vcvt_f32_f16(vget_low_f16(this->v)), sinLo, cosLo);
float32x4_t sinHi = sinLo, cosHi = cosLo;
if constexpr (Len * Packing > 4) {
Base::sincos_f32x4(vcvt_f32_f16(vget_high_f16(this->v)), sinHi, cosHi);
}
return {
VectorF16<Len, Packing>(vcombine_f16(vcvt_f16_f32(sinLo), vcvt_f16_f32(sinHi))),
VectorF16<Len, Packing>(vcombine_f16(vcvt_f16_f32(cosLo), vcvt_f16_f32(cosHi)))
};
}
constexpr VectorF16<Len, Packing> Cos() const { return std::get<1>(SinCos()); }
constexpr VectorF16<Len, Packing> Sin() const { return std::get<0>(SinCos()); }
template <std::array<bool, Len> values>
constexpr VectorF16<Len, Packing> Negate() const {
constexpr auto mask = []() {
std::array<std::uint16_t, NElems> m{};
for (std::uint8_t p = 0; p < Packing; ++p)
for (std::uint8_t i = 0; i < Len; ++i)
m[p * Len + i] = values[i] ? 0x8000u : 0u;
return m;
}();
return VectorF16<Len, Packing>(vreinterpretq_f16_u16(veorq_u16(vreinterpretq_u16_f16(this->v), vld1q_u16(mask.data()))));
}
// vfmaq_f16(a, b, c) is a + b*c.
static constexpr VectorF16<Len, Packing> MulitplyAdd(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b, VectorF16<Len, Packing> add) {
return VectorF16<Len, Packing>(vfmaq_f16(add.v, a.v, b.v));
}
static constexpr VectorF16<Len, Packing> MulitplySub(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b, VectorF16<Len, Packing> sub) {
return VectorF16<Len, Packing>(vfmaq_f16(vnegq_f16(sub.v), a.v, b.v));
}
// a × b = (a * b.yzx - b * a.yzx).yzx, one shuffle less than building
// both yzx and zxy of each input.
constexpr static VectorF16<Len, Packing> Cross(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b) requires(Len == 3) {
VectorF16<Len, Packing> aYzx = a.template Shuffle<{{1,2,0}}>();
VectorF16<Len, Packing> bYzx = b.template Shuffle<{{1,2,0}}>();
VectorF16<Len, Packing> zxy(vfmaq_f16(vmulq_f16(a.v, bYzx.v), vnegq_f16(b.v), aYzx.v));
return zxy.template Shuffle<{{1,2,0}}>();
}
constexpr static _Float16 Dot(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b) requires(Packing == 1) {
float16x8_t sum = ToSlots(vmulq_f16(a.v, b.v));
for (std::uint8_t width = Slot; width > 1; width /= 2) sum = vpaddq_f16(sum, sum);
return vgetq_lane_f16(sum, 0);
}
template <const std::array<std::uint8_t, Len> ShuffleValues>
constexpr VectorF16<Len, Packing> Shuffle() const {
return VectorF16<Len, Packing>(Permute<GetShuffleLanes<ShuffleValues>()>(this->v, this->v));
}
template <std::array<bool, Len> ShuffleValues>
constexpr static VectorF16<Len, Packing> Blend(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b) {
return VectorF16<Len, Packing>(Permute<GetBlendLanes<ShuffleValues>()>(a.v, b.v));
}
template<typename... Rest>
requires((std::is_same_v<Rest, VectorF16<Len, Packing>> && ...) &&
(1 + sizeof...(Rest) == VectorBase<Len, Packing, _Float16>::BatchSize))
constexpr static auto LengthSq(VectorF16<Len, Packing> first, Rest... rest) {
constexpr std::uint8_t N = VectorBase<Len, Packing, _Float16>::BatchSize;
std::array<VectorF16<Len, Packing>, N> args{ first, rest... };
std::array<float16x8_t, N> squares;
for (std::uint8_t i = 0; i < N; ++i) squares[i] = vmulq_f16(args[i].v, args[i].v);
return VectorF16<1, static_cast<std::uint8_t>(Packing * N)>(SumPacks(squares));
}
template<typename... Rest>
requires((std::is_same_v<Rest, VectorF16<Len, Packing>> && ...) &&
(1 + sizeof...(Rest) == VectorBase<Len, Packing, _Float16>::BatchSize))
constexpr static auto Length(VectorF16<Len, Packing> first, Rest... rest) {
auto sq = LengthSq(first, rest...);
sq.v = vsqrtq_f16(sq.v);
return sq;
}
// Pairwise dot products across BatchSize pairs: Dot(a0, b0, a1, b1,
// ...) returns one lane per sub-vector pair.
template<typename... Rest>
requires((std::is_same_v<Rest, VectorF16<Len, Packing>> && ...) &&
(1 + sizeof...(Rest) == 2 * VectorBase<Len, Packing, _Float16>::BatchSize))
constexpr static auto Dot(VectorF16<Len, Packing> first, Rest... rest) {
constexpr std::uint8_t N = VectorBase<Len, Packing, _Float16>::BatchSize;
std::array<VectorF16<Len, Packing>, 2 * N> args{ first, rest... };
std::array<float16x8_t, N> products;
for (std::uint8_t i = 0; i < N; ++i) products[i] = vmulq_f16(args[2 * i].v, args[2 * i + 1].v);
return VectorF16<1, static_cast<std::uint8_t>(Packing * N)>(SumPacks(products));
}
// One sqrt and one divide cover the whole batch; each input then
// multiplies by its own lanes of the result.
template<typename... Rest>
requires((std::is_same_v<Rest, VectorF16<Len, Packing>> && ...) &&
(1 + sizeof...(Rest) == VectorBase<Len, Packing, _Float16>::BatchSize))
constexpr static auto Normalize(VectorF16<Len, Packing> first, Rest... rest) {
constexpr std::uint8_t N = VectorBase<Len, Packing, _Float16>::BatchSize;
std::array<VectorF16<Len, Packing>, N> args{ first, rest... };
float16x8_t scale = vdivq_f16(vdupq_n_f16(1), Length(first, rest...).v);
return [&]<std::size_t... Is>(std::index_sequence<Is...>) {
return std::array<VectorF16<Len, Packing>, N>{
VectorF16<Len, Packing>(vmulq_f16(args[Is].v, Permute<GetSpreadLanes<Is>()>(scale, scale)))...
};
}(std::make_index_sequence<N>{});
}
constexpr static VectorF16<Len, Packing> Rotate(VectorF16<3, Packing> v, VectorF16<4, Packing> q) requires(Len == 3) {
VectorF16<3, Packing> qv = q;
VectorF16<3, Packing> qw = q.template Shuffle<{{3,3,3,3}}>();
VectorF16<3, Packing> t = Cross(qv, v) * _Float16(2);
return MulitplyAdd(t, qw, v) + Cross(qv, t);
}
constexpr static VectorF16<3, Packing> RotatePivot(VectorF16<3, Packing> v, VectorF16<4, Packing> q, VectorF16<3, Packing> pivot) requires(Len == 3) {
VectorF16<3, Packing> translated = v - pivot;
return Rotate(translated, q) + pivot;
}
// With (roll, pitch, yaw) half angles in x, y, z:
// x = sx*cy*cz - cx*sy*sz y = cx*sy*cz + sx*cy*sz
// z = cx*cy*sz - sx*sy*cz w = cx*cy*cz + sx*sy*sz
// Each factor row below is one two-source shuffle of the sin and cos
// registers (see GetEulerLanes).
constexpr static VectorF16<4, Packing> QuanternionFromEuler(VectorF16<3, Packing> EulerHalf) requires(Len == 4) {
std::tuple<VectorF16<3, Packing>, VectorF16<3, Packing>> sinCos = EulerHalf.SinCos();
float16x8_t s = std::get<0>(sinCos).v;
float16x8_t c = std::get<1>(sinCos).v;
float16x8_t left = vmulq_f16(
vmulq_f16(
Permute<GetEulerLanes<0, false>()>(s, c), // sx cx cx cx
Permute<GetEulerLanes<1, false>()>(s, c)), // cy sy cy cy
Permute<GetEulerLanes<2, false>()>(s, c)); // cz cz sz cz
float16x8_t right = vmulq_f16(
Permute<GetEulerLanes<0, true>()>(s, c), // cx sx sx sx
Permute<GetEulerLanes<1, true>()>(s, c)); // sy cy sy sy
VectorF16<4, Packing> rightZ(Permute<GetEulerLanes<2, true>()>(s, c)); // sz sz cz sz
rightZ = rightZ.template Negate<{{true, false, true, false}}>();
return VectorF16<4, Packing>(vfmaq_f16(left, right, rightZ.v));
}
};
}
#elif !defined(__x86_64) && defined(__FLT16_MAX__) && !(defined(__aarch64__) && defined(__ARM_NEON))
// Scalar software fallback for non-x86_64 targets that still have _Float16.
namespace Crafter {
export template <std::uint8_t Len, std::uint8_t Packing>
struct VectorF16 : public VectorBase<Len, Packing, _Float16> {
template <std::uint8_t Len2, std::uint8_t Packing2>
friend struct VectorF16;
using Base = VectorBase<Len, Packing, _Float16>;
static constexpr std::uint8_t NElems = Base::AlignmentElement;
constexpr VectorF16() = default;
constexpr VectorF16(typename Base::VectorType vv) {
this->v = vv;
}
constexpr VectorF16(const _Float16* vB) { Load(vB); }
constexpr VectorF16(_Float16 val) {
for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = val;
}
constexpr void Load(const _Float16* vB) {
for (std::uint8_t i = 0; i < NElems; ++i) this->v[i] = vB[i];
}
constexpr void Store(_Float16* vB) const {
for (std::uint8_t i = 0; i < NElems; ++i) vB[i] = this->v[i];
}
template<typename T>
constexpr std::array<_Float16, NElems> Store() const {
std::array<_Float16, NElems> r{};
Store(r.data());
return r;
}
template <std::uint8_t BLen, std::uint8_t BPacking>
constexpr operator VectorF16<BLen, BPacking>() const {
VectorF16<BLen, BPacking> r;
const std::uint8_t copyLen = (BLen < Len) ? BLen : Len;
const std::uint8_t copyPack = (BPacking < Packing) ? BPacking : Packing;
for (std::uint8_t p = 0; p < copyPack; ++p)
for (std::uint8_t i = 0; i < copyLen; ++i)
r.v[p * BLen + i] = this->v[p * Len + i];
return r;
}
constexpr VectorF16<Len, Packing> operator+(VectorF16<Len, Packing> b) const {
VectorF16<Len, Packing> r;
for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] + b.v[i];
return r;
}
constexpr VectorF16<Len, Packing> operator-(VectorF16<Len, Packing> b) const {
VectorF16<Len, Packing> r;
for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] - b.v[i];
return r;
}
constexpr VectorF16<Len, Packing> operator*(VectorF16<Len, Packing> b) const {
VectorF16<Len, Packing> r;
for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] * b.v[i];
return r;
}
constexpr VectorF16<Len, Packing> operator/(VectorF16<Len, Packing> b) const {
VectorF16<Len, Packing> r;
for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = this->v[i] / b.v[i];
return r;
}
constexpr void operator+=(VectorF16<Len, Packing> b) { for (std::uint8_t i=0;i<NElems;++i) this->v[i] += b.v[i]; }
constexpr void operator-=(VectorF16<Len, Packing> b) { for (std::uint8_t i=0;i<NElems;++i) this->v[i] -= b.v[i]; }
constexpr void operator*=(VectorF16<Len, Packing> b) { for (std::uint8_t i=0;i<NElems;++i) this->v[i] *= b.v[i]; }
constexpr void operator/=(VectorF16<Len, Packing> b) { for (std::uint8_t i=0;i<NElems;++i) this->v[i] /= b.v[i]; }
constexpr VectorF16<Len, Packing> operator+(_Float16 b) const { return *this + VectorF16<Len, Packing>(b); }
constexpr VectorF16<Len, Packing> operator-(_Float16 b) const { return *this - VectorF16<Len, Packing>(b); }
constexpr VectorF16<Len, Packing> operator*(_Float16 b) const { return *this * VectorF16<Len, Packing>(b); }
constexpr VectorF16<Len, Packing> operator/(_Float16 b) const { return *this / VectorF16<Len, Packing>(b); }
constexpr void operator+=(_Float16 b) { *this += VectorF16<Len, Packing>(b); }
constexpr void operator-=(_Float16 b) { *this -= VectorF16<Len, Packing>(b); }
constexpr void operator*=(_Float16 b) { *this *= VectorF16<Len, Packing>(b); }
constexpr void operator/=(_Float16 b) { *this /= VectorF16<Len, Packing>(b); }
constexpr VectorF16<Len, Packing> operator-() const {
VectorF16<Len, Packing> r;
for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = -this->v[i];
return r;
}
constexpr bool operator==(VectorF16<Len, Packing> b) const {
for (std::uint8_t p = 0; p < Packing; ++p)
for (std::uint8_t i = 0; i < Len; ++i)
if (this->v[p * Len + i] != b.v[p * Len + i]) return false;
return true;
}
constexpr bool operator!=(VectorF16<Len, Packing> b) const { return !(*this == b); }
template<std::uint32_t ExtractLen>
constexpr VectorF16<ExtractLen, Packing> ExtractLo() const {
VectorF16<ExtractLen, Packing> r;
for (std::uint8_t p = 0; p < Packing; ++p)
for (std::uint8_t i = 0; i < ExtractLen; ++i)
r.v[p * ExtractLen + i] = this->v[p * Len + i];
return r;
}
// Transcendentals are computed via float since libstdc++ doesn't always
// provide overloads for _Float16; the result is rounded back to half.
constexpr VectorF16<Len, Packing> Cos() const {
VectorF16<Len, Packing> r;
for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = static_cast<_Float16>(std::cos(static_cast<float>(this->v[i])));
return r;
}
constexpr VectorF16<Len, Packing> Sin() const {
VectorF16<Len, Packing> r;
for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = static_cast<_Float16>(std::sin(static_cast<float>(this->v[i])));
return r;
}
constexpr std::tuple<VectorF16<Len, Packing>, VectorF16<Len, Packing>> SinCos() const {
return { Sin(), Cos() };
}
template <std::array<bool, Len> values>
constexpr VectorF16<Len, Packing> Negate() const {
VectorF16<Len, Packing> r;
for (std::uint8_t p = 0; p < Packing; ++p)
for (std::uint8_t i = 0; i < Len; ++i)
r.v[p * Len + i] = values[i] ? -this->v[p * Len + i] : this->v[p * Len + i];
return r;
}
static constexpr VectorF16<Len, Packing> MulitplyAdd(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b, VectorF16<Len, Packing> add) {
VectorF16<Len, Packing> r;
for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = a.v[i] * b.v[i] + add.v[i];
return r;
}
static constexpr VectorF16<Len, Packing> MulitplySub(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b, VectorF16<Len, Packing> sub) {
VectorF16<Len, Packing> r;
for (std::uint8_t i = 0; i < NElems; ++i) r.v[i] = a.v[i] * b.v[i] - sub.v[i];
return r;
}
constexpr static VectorF16<Len, Packing> Cross(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b) requires(Len == 3) {
VectorF16<Len, Packing> r;
for (std::uint8_t p = 0; p < Packing; ++p) {
const std::uint8_t base = p * 3;
r.v[base + 0] = a.v[base + 1] * b.v[base + 2] - a.v[base + 2] * b.v[base + 1];
r.v[base + 1] = a.v[base + 2] * b.v[base + 0] - a.v[base + 0] * b.v[base + 2];
r.v[base + 2] = a.v[base + 0] * b.v[base + 1] - a.v[base + 1] * b.v[base + 0];
}
return r;
}
template <const std::array<std::uint8_t, Len> ShuffleValues>
constexpr VectorF16<Len, Packing> Shuffle() const {
VectorF16<Len, Packing> r;
for (std::uint8_t p = 0; p < Packing; ++p)
for (std::uint8_t i = 0; i < Len; ++i)
r.v[p * Len + i] = this->v[p * Len + ShuffleValues[i]];
return r;
}
template <std::array<bool, Len> ShuffleValues>
constexpr static VectorF16<Len, Packing> Blend(VectorF16<Len, Packing> a, VectorF16<Len, Packing> b) {
VectorF16<Len, Packing> r;
for (std::uint8_t p = 0; p < Packing; ++p)
for (std::uint8_t i = 0; i < Len; ++i)
r.v[p * Len + i] = ShuffleValues[i] ? b.v[p * Len + i] : a.v[p * Len + i];
return r;
}
template<typename... Rest>
requires((std::is_same_v<Rest, VectorF16<Len, Packing>> && ...) &&
(1 + sizeof...(Rest) == VectorBase<Len, Packing, _Float16>::BatchSize))
constexpr static auto LengthSq(VectorF16<Len, Packing> first, Rest... rest) {
constexpr std::uint8_t N = VectorBase<Len, Packing, _Float16>::BatchSize;
VectorF16<1, static_cast<std::uint8_t>(Packing * N)> r;
std::array<VectorF16<Len, Packing>, N> args{ first, rest... };
for (std::uint8_t i = 0; i < N; ++i)
for (std::uint8_t p = 0; p < Packing; ++p) {
_Float16 acc = _Float16(0);
for (std::uint8_t k = 0; k < Len; ++k) {
_Float16 x = args[i].v[p * Len + k];
acc += x * x;
}
r.v[i * Packing + p] = acc;
}
return r;
}
template<typename... Rest>
requires((std::is_same_v<Rest, VectorF16<Len, Packing>> && ...) &&
(1 + sizeof...(Rest) == VectorBase<Len, Packing, _Float16>::BatchSize))
constexpr static auto Length(VectorF16<Len, Packing> first, Rest... rest) {
auto sq = LengthSq(first, rest...);
for (std::uint8_t i = 0; i < decltype(sq)::NElems; ++i)
sq.v[i] = static_cast<_Float16>(std::sqrt(static_cast<float>(sq.v[i])));
return sq;
}
template<typename... Rest>
requires((std::is_same_v<Rest, VectorF16<Len, Packing>> && ...) &&
(1 + sizeof...(Rest) == VectorBase<Len, Packing, _Float16>::BatchSize))
constexpr static auto Normalize(VectorF16<Len, Packing> first, Rest... rest) {
auto normOne = [](VectorF16<Len, Packing> u) {
VectorF16<Len, Packing> out;
for (std::uint8_t p = 0; p < Packing; ++p) {
float acc = 0.0f;
for (std::uint8_t k = 0; k < Len; ++k) {
float x = static_cast<float>(u.v[p * Len + k]);
acc += x * x;
}
_Float16 invLen = acc > 0.0f ? static_cast<_Float16>(1.0f / std::sqrt(acc)) : _Float16(0);
for (std::uint8_t k = 0; k < Len; ++k)
out.v[p * Len + k] = u.v[p * Len + k] * invLen;
}
return out;
};
return std::array<VectorF16<Len, Packing>, VectorBase<Len, Packing, _Float16>::BatchSize>{ normOne(first), normOne(rest)... };
}
constexpr static VectorF16<Len, Packing> Rotate(VectorF16<3, Packing> v, VectorF16<4, Packing> q) requires(Len == 3) {
VectorF16<3, Packing> qv;
VectorF16<3, Packing> qwBroadcast;
for (std::uint8_t p = 0; p < Packing; ++p) {
qv.v[p * 3 + 0] = q.v[p * 4 + 0];
qv.v[p * 3 + 1] = q.v[p * 4 + 1];
qv.v[p * 3 + 2] = q.v[p * 4 + 2];
for (std::uint8_t i = 0; i < 3; ++i) qwBroadcast.v[p * 3 + i] = q.v[p * 4 + 3];
}
VectorF16<3, Packing> t = Cross(qv, v) * _Float16(2);
return v + t * qwBroadcast + Cross(qv, t);
}
constexpr static VectorF16<3, Packing> RotatePivot(VectorF16<3, Packing> v, VectorF16<4, Packing> q, VectorF16<3, Packing> pivot) requires(Len == 3) {
VectorF16<3, Packing> translated = v - pivot;
return Rotate(translated, q) + pivot;
}
constexpr static VectorF16<4, Packing> QuanternionFromEuler(VectorF16<3, Packing> eulerHalf) requires(Len == 4) {
VectorF16<4, Packing> r;
for (std::uint8_t p = 0; p < Packing; ++p) {
float roll = static_cast<float>(eulerHalf.v[p * 3 + 0]);
float pitch = static_cast<float>(eulerHalf.v[p * 3 + 1]);
float yaw = static_cast<float>(eulerHalf.v[p * 3 + 2]);
float sr = std::sin(roll), cr = std::cos(roll);
float sp = std::sin(pitch), cp = std::cos(pitch);
float sy = std::sin(yaw), cy = std::cos(yaw);
r.v[p * 4 + 0] = static_cast<_Float16>(sr * cp * cy - cr * sp * sy);
r.v[p * 4 + 1] = static_cast<_Float16>(cr * sp * cy + sr * cp * sy);
r.v[p * 4 + 2] = static_cast<_Float16>(cr * cp * sy - sr * sp * cy);
r.v[p * 4 + 3] = static_cast<_Float16>(cr * cp * cy + sr * sp * sy);
}
return r;
}
};
}
#endif
#if defined(__FLT16_MAX__) && (!defined(__x86_64) || defined(__AVX512FP16__)) && !(defined(__aarch64__) && defined(__ARM_NEON) && !defined(__ARM_FEATURE_FP16_VECTOR_ARITHMETIC))
export template <std::uint32_t Len, std::uint32_t Packing>
struct std::formatter<Crafter::VectorF16<Len, Packing>> : std::formatter<std::string> {
constexpr auto format(const Crafter::VectorF16<Len, Packing>& obj, format_context& ctx) const {
std::array<_Float16, Crafter::VectorF16<Len, Packing>::AlignmentElement> vec = obj.template Store<float>();
std::string out = "{";
for(std::uint32_t i = 0; i < Packing; i++) {
out += "{";
for(std::uint32_t i2 = 0; i2 < Len; i2++) {
out += std::format("{}", static_cast<float>(vec[i * Len + i2]));
if (i2 + 1 < Len) out += ",";
}
out += "}";
}
out += "}";
return std::formatter<std::string>::format(out, ctx);
}
};
#endif
Reference in a new issue View git blame Copy permalink