Catcrafts/Crafter.DB
Watch
2
0
Fork
You've already forked Crafter.DB
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial commit: Crafter.DB

Browse source 
Header-only C++26 module library for typed, file-backed tables with
schema reflection, migrations, queries, and crash-safe commits.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Jorijn van der Graaf 2026-10-17 04:16:51 +02:00
commit bfd0cc261b
Signed by: jorijnvdgraaf
GPG key ID: 2937E59CDCC1BCFB
34 changed files with 7506 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

335
interfaces/Crafter.DB-Codec.cppm Normal file
View file

@ -0,0 +1,335 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// Typed encode/decode against a Layout<T>. Every function here is a template
// over (table type, first leaf index, member type), so column offsets are
// compile-time constants and a scalar read is one load. Leaf indices follow
// declaration order, depth first, exactly as Schema enumerates them.
export module Crafter.DB:Codec;
import std;
import :Reflect;
import :Format;
import :Formatter;
import :Schema;
import :Storage;
import :Error;
export namespace Crafter::DB {
// What a Row hands out for a member of type M: the formatter's View for
// variable-length members, an optional of that for std::optional, and the
// value itself for everything else.
template<class M>
consteval auto ViewTypeOf() {
constexpr Shape shape = ShapeOf<M>();
if constexpr (shape == Shape::Var) return std::type_identity<typename Formatter<M>::View>{};
else if constexpr (shape == Shape::Optional) return std::type_identity<std::optional<typename decltype(ViewTypeOf<typename OptionalTraits<M>::Type>())::type>>{};
else return std::type_identity<M>{};
}
template<class M>
using View = typename decltype(ViewTypeOf<M>())::type;
// Zeroes the columns of leaves [First, First + Count): the payload of a
// disengaged std::optional must not keep stale bytes (or stale heap
// handles that Compact would then preserve).
template<class T, std::size_t First, std::size_t Count>
void ZeroLeaves(std::byte* row) noexcept {
[&]<std::size_t... I>(std::index_sequence<I...>) {
(std::memset(row + Layout<T>::Offset(First + I), 0, Layout<T>::columns[Layout<T>::Column(First + I)].size), ...);
}(std::make_index_sequence<Count>{});
}
// Copies the columns of leaves [First, First + Count) from one row image
// to another, leaving every other byte (the flags byte above all) alone.
template<class T, std::size_t First, std::size_t Count>
void CopyLeaves(std::byte* destination, const std::byte* source) noexcept {
[&]<std::size_t... I>(std::index_sequence<I...>) {
(std::memcpy(destination + Layout<T>::Offset(First + I), source + Layout<T>::Offset(First + I), Layout<T>::columns[Layout<T>::Column(First + I)].size), ...);
}(std::make_index_sequence<Count>{});
}
// Values are converted implicitly, never static_cast: an int does not
// silently become an enum, and swapped Emplace arguments do not compile.
template<class T, std::size_t Leaf, class M, class V>
void EncodeInto(std::byte* row, HeapWriter& heap, const V& value) {
constexpr Shape shape = ShapeOf<M>();
constexpr std::uint16_t offset = Layout<T>::Offset(Leaf);
if constexpr (shape == Shape::Scalar) {
const M scalar = value;
std::memcpy(row + offset, &scalar, sizeof(M));
} else if constexpr (shape == Shape::FixedBytes) {
if constexpr (FixedStringTraits<M>::value) {
const M fixed(value);
std::memcpy(row + offset, fixed.chars.data(), fixed.chars.size());
} else {
const M& array = value;
std::memcpy(row + offset, array.data(), sizeof(M));
}
} else if constexpr (shape == Shape::ScalarArray) {
using E = typename StdArrayTraits<M>::Element;
[&]<std::size_t... I>(std::index_sequence<I...>) {
(EncodeInto<T, Leaf + I, E>(row, heap, value[I]), ...);
}(std::make_index_sequence<StdArrayTraits<M>::size>{});
} else if constexpr (shape == Shape::Optional) {
using P = typename OptionalTraits<M>::Type;
const std::uint8_t present = value.has_value() ? 1 : 0;
std::memcpy(row + offset, &present, 1);
if (value.has_value()) EncodeInto<T, Leaf + 1, P>(row, heap, *value);
else ZeroLeaves<T, Leaf + 1, LeafCount<P>()>(row);
} else if constexpr (shape == Shape::Var) {
const Var var = Formatter<M>::Encode(value, heap);
std::memcpy(row + offset, &var, sizeof(var));
} else if constexpr (shape == Shape::Fixed) {
using S = typename Formatter<M>::Stored;
if constexpr (std::is_same_v<V, M>) {
const S stored = Formatter<M>::Encode(value);
EncodeInto<T, Leaf, S>(row, heap, stored);
} else {
const M converted = value;
const S stored = Formatter<M>::Encode(converted);
EncodeInto<T, Leaf, S>(row, heap, stored);
}
} else if constexpr (shape == Shape::Aggregate) {
if constexpr (std::is_same_v<V, M>) {
[&]<std::size_t... I>(std::index_sequence<I...>) {
(EncodeInto<T, Leaf + LeafStart<M, I>(), MemberType<I, M>>(row, heap, Member<I>(value)), ...);
}(std::make_index_sequence<memberCount<M>>{});
} else {
const M converted = value;
EncodeInto<T, Leaf, M>(row, heap, converted);
}
} else {
static_assert(shape == Shape::Scalar, "unsupported member shape reached the encoder; CheckTable should have rejected it");
}
}
template<class T, std::size_t Leaf, class M>
M DecodeValue(const std::byte* row, const Heap& heap);
template<class T, std::size_t Leaf, class M>
View<M> DecodeView(const std::byte* row, const Heap& heap) {
constexpr Shape shape = ShapeOf<M>();
constexpr std::uint16_t offset = Layout<T>::Offset(Leaf);
if constexpr (shape == Shape::Var) {
return Formatter<M>::Decode(row + offset, heap);
} else if constexpr (shape == Shape::Optional) {
using P = typename OptionalTraits<M>::Type;
std::uint8_t present;
std::memcpy(&present, row + offset, 1);
if (present == 0) return std::nullopt;
return View<M>(DecodeView<T, Leaf + 1, P>(row, heap));
} else {
return DecodeValue<T, Leaf, M>(row, heap);
}
}
template<class T, std::size_t Leaf, class M>
M DecodeValue(const std::byte* row, const Heap& heap) {
constexpr Shape shape = ShapeOf<M>();
constexpr std::uint16_t offset = Layout<T>::Offset(Leaf);
if constexpr (shape == Shape::Scalar) {
M scalar;
std::memcpy(&scalar, row + offset, sizeof(M));
return scalar;
} else if constexpr (shape == Shape::FixedBytes) {
M fixed{};
if constexpr (FixedStringTraits<M>::value) std::memcpy(fixed.chars.data(), row + offset, fixed.chars.size());
else std::memcpy(fixed.data(), row + offset, sizeof(M));
return fixed;
} else if constexpr (shape == Shape::ScalarArray) {
using E = typename StdArrayTraits<M>::Element;
M array{};
[&]<std::size_t... I>(std::index_sequence<I...>) {
((array[I] = DecodeValue<T, Leaf + I, E>(row, heap)), ...);
}(std::make_index_sequence<StdArrayTraits<M>::size>{});
return array;
} else if constexpr (shape == Shape::Optional) {
using P = typename OptionalTraits<M>::Type;
std::uint8_t present;
std::memcpy(&present, row + offset, 1);
if (present == 0) return std::nullopt;
return M(DecodeValue<T, Leaf + 1, P>(row, heap));
} else if constexpr (shape == Shape::Var) {
return MaterializeVar<M>(Formatter<M>::Decode(row + offset, heap));
} else if constexpr (shape == Shape::Fixed) {
using S = typename Formatter<M>::Stored;
return Formatter<M>::Decode(DecodeValue<T, Leaf, S>(row, heap));
} else if constexpr (shape == Shape::Aggregate) {
M aggregate{};
[&]<std::size_t... I>(std::index_sequence<I...>) {
((Member<I>(aggregate) = DecodeValue<T, Leaf + LeafStart<M, I>(), MemberType<I, M>>(row, heap)), ...);
}(std::make_index_sequence<memberCount<M>>{});
return aggregate;
} else {
static_assert(shape == Shape::Scalar, "unsupported member shape reached the decoder");
}
}
// --- row level -----------------------------------------------------------
template<Aggregate T, std::size_t I>
View<MemberType<I, T>> DecodeMember(const std::byte* row, const Heap& heap) {
return DecodeView<T, LeafStart<T, I>(), MemberType<I, T>>(row, heap);
}
template<Aggregate T, std::size_t I>
MemberType<I, T> DecodeMemberValue(const std::byte* row, const Heap& heap) {
return DecodeValue<T, LeafStart<T, I>(), MemberType<I, T>>(row, heap);
}
template<Aggregate T>
T DecodeRow(const std::byte* row, const Heap& heap) {
T value{};
[&]<std::size_t... I>(std::index_sequence<I...>) {
((Member<I>(value) = DecodeMemberValue<T, I>(row, heap)), ...);
}(std::make_index_sequence<memberCount<T>>{});
return value;
}
// Writes a live row (flags = 0) for `value` into `row`, which must be
// Layout<T>::rowSize bytes of writable memory.
template<Aggregate T>
void EncodeRow(const T& value, std::byte* row, HeapWriter& heap) {
std::memset(row, 0, Layout<T>::rowSize);
[&]<std::size_t... I>(std::index_sequence<I...>) {
(EncodeInto<T, LeafStart<T, I>(), MemberType<I, T>>(row, heap, Member<I>(value)), ...);
}(std::make_index_sequence<memberCount<T>>{});
}
// Whether an Emplace argument of type A is acceptable for a member of type
// M: what the formatter's Encode accepts for variable-length members, an
// implicit conversion for everything else.
template<class M, class A>
consteval bool EmplaceAccepts() {
constexpr Shape shape = ShapeOf<M>();
if constexpr (shape == Shape::Var) return requires(const A& a, HeapWriter& w) { Formatter<M>::Encode(a, w); };
else if constexpr (shape == Shape::FixedBytes && FixedStringTraits<M>::value) return std::is_constructible_v<M, const A&>;
else return std::is_convertible_v<const A&, M>;
}
template<Aggregate T, std::size_t I, class A>
consteval bool CheckEmplaceArgument() {
static_assert(EmplaceAccepts<MemberType<I, T>, A>(), Message({ "Crafter.DB: the Emplace argument for member '", MemberName<I, T>(), "' of ", TypeName<T>(), " has type ", TypeName<A>(), ", which does not convert to the member; arguments follow declaration order." }));
return true;
}
// One argument per member, each encoded straight into its columns.
template<Aggregate T, class... Args>
void EmplaceRow(std::byte* row, HeapWriter& heap, const Args&... args) {
static_assert(sizeof...(Args) == memberCount<T>, "Crafter.DB: Emplace takes exactly one argument per member of the table struct, in declaration order");
[&]<std::size_t... I>(std::index_sequence<I...>) {
(CheckEmplaceArgument<T, I, Args>(), ...);
}(std::make_index_sequence<memberCount<T>>{});
std::memset(row, 0, Layout<T>::rowSize);
[&]<std::size_t... I>(std::index_sequence<I...>) {
(EncodeInto<T, LeafStart<T, I>(), MemberType<I, T>>(row, heap, args), ...);
}(std::make_index_sequence<memberCount<T>>{});
}
// Encodes member I only (Set); the rest of the row is left alone.
template<Aggregate T, std::size_t I, class V>
void EncodeMember(std::byte* row, HeapWriter& heap, const V& value) {
EncodeInto<T, LeafStart<T, I>(), MemberType<I, T>>(row, heap, value);
}
// Out-of-line heap bytes referenced by member I's columns.
template<Aggregate T, std::size_t I>
std::uint64_t MemberHeapBytes(const std::byte* row) noexcept {
std::uint64_t bytes = 0;
constexpr std::size_t first = LeafStart<T, I>();
constexpr std::size_t count = LeafCount<MemberType<I, T>>();
for (std::size_t leaf = first; leaf < first + count; leaf++) {
const ColumnInfo& column = Layout<T>::columns[Layout<T>::Column(leaf)];
if (column.kind != Kind::String && column.kind != Kind::Blob) continue;
Var var;
std::memcpy(&var, row + column.offset, sizeof(var));
if (!VarInline(var.length, column.kind == Kind::Blob && column.paramNum != 0 ? column.paramNum : 1)) bytes += var.length;
}
return bytes;
}
// --- ordered column ------------------------------------------------------
template<class T>
using OrderedKey = MemberTypeOf<T::orderedBy>;
template<class T> requires HasOrderedBy<T>
OrderedKey<T> KeyOfSlot(const TableFile& file, std::uint64_t slot) {
return DecodeMemberValue<T, Layout<T>::orderedMember>(file.Slot(slot), file.HeapView());
}
template<class T> requires HasOrderedBy<T>
std::uint64_t LowerBoundSlot(const TableFile& file, const OrderedKey<T>& key, std::uint64_t first, std::uint64_t last) {
while (first < last) {
const std::uint64_t mid = first + (last - first) / 2;
if (KeyOfSlot<T>(file, mid) < key) first = mid + 1;
else last = mid;
}
return first;
}
template<class T> requires HasOrderedBy<T>
std::uint64_t UpperBoundSlot(const TableFile& file, const OrderedKey<T>& key, std::uint64_t first, std::uint64_t last) {
while (first < last) {
const std::uint64_t mid = first + (last - first) / 2;
if (key < KeyOfSlot<T>(file, mid)) last = mid;
else first = mid + 1;
}
return first;
}
// Throws OrderError when `key` would sit below the key of slot `before`
// (or above the key of slot `after`, when given).
template<class T> requires HasOrderedBy<T>
void CheckOrder(const TableFile& file, const OrderedKey<T>& key, std::optional<std::uint64_t> before, std::optional<std::uint64_t> after) {
if (before && key < KeyOfSlot<T>(file, *before)) {
throw OrderError(std::format("{}: value would break the order of '{}' (it sorts before the previous row)", file.Path().string(), MemberName<Layout<T>::orderedMember, T>()));
}
if (after && KeyOfSlot<T>(file, *after) < key) {
throw OrderError(std::format("{}: value would break the order of '{}' (it sorts after the next row)", file.Path().string(), MemberName<Layout<T>::orderedMember, T>()));
}
}
// Appends encoded rows. Checks the ordered column against the last slot
// and within the batch before anything is written.
template<Aggregate T, std::ranges::input_range R> requires std::convertible_to<std::ranges::range_reference_t<R>, const T&>
std::uint64_t AppendRows(TableFile& file, R&& values) {
file.RequireWritable();
const std::uint64_t first = file.SlotCount();
if constexpr (std::ranges::sized_range<R> && std::ranges::random_access_range<R>) {
const std::uint64_t count = std::ranges::size(values);
if (count == 0) return first;
if constexpr (HasOrderedBy<T>) {
std::optional<std::uint64_t> before;
if (first != 0) before = first - 1;
auto it = std::ranges::begin(values);
CheckOrder<T>(file, static_cast<const T&>(*it).*T::orderedBy, before, std::nullopt);
for (std::uint64_t i = 1; i < count; i++) {
if (static_cast<const T&>(it[i]).*T::orderedBy < static_cast<const T&>(it[i - 1]).*T::orderedBy) {
throw OrderError(std::format("{}: batch is not sorted by '{}'", file.Path().string(), MemberName<Layout<T>::orderedMember, T>()));
}
}
}
std::byte* out = file.BeginAppend(count);
HeapWriter& heap = file.Writer();
try {
for (const T& value : values) {
EncodeRow<T>(value, out, heap);
out += Layout<T>::rowSize;
}
} catch (...) {
file.AbortAppend();
throw;
}
file.CommitAppend(count);
return first;
} else {
// A range that cannot be sized is buffered first, so the order
// check runs over the whole batch before any row is written and a
// failing batch leaves the table untouched.
std::vector<T> buffered;
for (const T& value : values) buffered.push_back(value);
return AppendRows<T>(file, std::span<const T>(buffered));
}
}
}

65
interfaces/Crafter.DB-Error.cppm Normal file
View file

@ -0,0 +1,65 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
export module Crafter.DB:Error;
import std;
export namespace Crafter::DB {
// Root of every exception the library throws. Catch this to handle
// "the table is unusable" uniformly; catch the subclasses to react.
class Error : public std::runtime_error {
public:
using std::runtime_error::runtime_error;
};
// A system call failed. `code` carries errno, `path` the file involved.
class IOError : public Error {
public:
IOError(std::string_view what, std::error_code code, std::filesystem::path path)
: Error(std::format("{}: {} ({})", path.string(), what, code.message())), code(code), path(std::move(path)) {}
std::error_code code;
std::filesystem::path path;
};
// The file is not a Crafter.DB table or its header/commit records fail
// validation (bad magic, both commit CRCs wrong, extents outside the file).
class CorruptError : public Error {
public:
using Error::Error;
};
// The compiled struct and the file disagree: same version but a different
// fingerprint (struct changed without bumping `version`), a file newer
// than the code, or no `Previous` chain reaching the file's version.
class SchemaError : public Error {
public:
using Error::Error;
};
// A migration is needed but cannot run: the table was opened read-only or
// with `migrate = false`.
class MigrationError : public Error {
public:
using Error::Error;
};
// Another process (or another Table object in this process) holds the
// file lock in a conflicting mode.
class LockedError : public Error {
public:
using Error::Error;
};
// A write would break the order of the `orderedBy` column.
class OrderError : public Error {
public:
using Error::Error;
};
// A write was attempted on a table opened with `readOnly = true`.
class ReadOnlyError : public Error {
public:
using Error::Error;
};
}

142
interfaces/Crafter.DB-File.cppm Normal file
View file

@ -0,0 +1,142 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// Everything that talks to the operating system: file descriptors, advisory
// locks, preallocation, the reserved virtual ranges that keep row pointers
// stable while a table grows, durability calls and the atomic create/replace
// dance. Nothing here knows what a table is; the Storage partition does.
export module Crafter.DB:File;
import std;
import :Error;
export namespace Crafter::DB {
enum class OpenMode : std::uint8_t {
OpenOrCreate, // open the table, create it when missing (default)
OpenExisting, // IOError when missing
CreateNew, // IOError when present
};
enum class Durability : std::uint8_t {
OnFlush, // durable after Flush() and on destruction (default)
EveryAppend, // Flush() after every Append/Emplace/Update/Set/Erase
};
struct OpenOptions {
OpenMode mode = OpenMode::OpenOrCreate;
bool readOnly = false; // PROT_READ mapping, shared lock, no migration, no writes
bool createDirectories = true; // create missing parent directories with directoryPermissions
std::filesystem::perms filePermissions = std::filesystem::perms::owner_read | std::filesystem::perms::owner_write; // 0600
std::filesystem::perms directoryPermissions = std::filesystem::perms::owner_all; // 0700
Durability durability = Durability::OnFlush;
bool migrate = true; // run the Previous chain on an older file; false throws MigrationError instead
bool backupBeforeMigrate = true; // keep the pre-migration file as <file>.v<old>.bak
std::uint64_t reserveBytes = std::uint64_t(32) << 30; // virtual address space reserved per region
bool excludeFromCoreDumps = true; // MADV_DONTDUMP on the mappings
bool prefetch = false; // MADV_WILLNEED over the committed data at open (cold-cache batch jobs)
};
// An open file descriptor. Every operation throws IOError on failure.
class File {
public:
File() = default;
File(const File&) = delete;
File& operator=(const File&) = delete;
File(File&& other) noexcept;
File& operator=(File&& other) noexcept;
~File();
static File Open(const std::filesystem::path& path, bool writable);
static File CreateExclusive(const std::filesystem::path& path, std::filesystem::perms permissions);
bool IsOpen() const noexcept { return descriptor != -1; }
std::int32_t Descriptor() const noexcept { return descriptor; }
const std::filesystem::path& Path() const noexcept { return path; }
void SetPath(std::filesystem::path newPath) { path = std::move(newPath); }
// flock(LOCK_NB). Advisory, per open file description, released on close.
void LockExclusive();
void LockShared();
std::uint64_t Size() const;
void Truncate(std::uint64_t length);
// fallocate: real blocks, so a full disk is an error here and never a
// SIGBUS on first touch of a mapped page.
void Allocate(std::uint64_t offset, std::uint64_t length);
void ReadAt(std::uint64_t offset, std::span<std::byte> out) const;
void WriteAt(std::uint64_t offset, std::span<const std::byte> in);
void SyncData();
void Sync();
void Close() noexcept;
private:
std::int32_t descriptor = -1;
std::filesystem::path path;
};
// A reserved virtual address range into which file extents are mapped
// back to back, so the region reads as one contiguous array that grows
// without ever moving.
class MappedRegion {
public:
MappedRegion() = default;
MappedRegion(const MappedRegion&) = delete;
MappedRegion& operator=(const MappedRegion&) = delete;
MappedRegion(MappedRegion&& other) noexcept;
MappedRegion& operator=(MappedRegion&& other) noexcept;
~MappedRegion();
void Reserve(std::uint64_t bytes);
// Maps [fileOffset, fileOffset + length) at Base() + Mapped().
void Map(const File& file, std::uint64_t fileOffset, std::uint64_t length, bool writable);
void ExcludeFromCoreDumps();
void WillNeed(std::uint64_t length);
void Release() noexcept;
std::byte* Base() const noexcept { return base; }
std::uint64_t Mapped() const noexcept { return mapped; }
std::uint64_t Reserved() const noexcept { return reserved; }
private:
std::byte* base = nullptr;
std::uint64_t reserved = 0;
std::uint64_t mapped = 0;
bool dontDump = false;
};
// A plain shared mapping of a few pages (the header).
class MappedPages {
public:
MappedPages() = default;
MappedPages(const MappedPages&) = delete;
MappedPages& operator=(const MappedPages&) = delete;
MappedPages(MappedPages&& other) noexcept;
MappedPages& operator=(MappedPages&& other) noexcept;
~MappedPages();
void Map(const File& file, std::uint64_t fileOffset, std::uint64_t length, bool writable);
void Sync();
void Release() noexcept;
std::byte* Data() const noexcept { return data; }
std::uint64_t Length() const noexcept { return length; }
private:
std::byte* data = nullptr;
std::uint64_t length = 0;
};
namespace Os {
// mkdir -p with an explicit mode for every created component, so a
// private cache directory is never world-readable, not even briefly.
void CreateDirectories(const std::filesystem::path& directory, std::filesystem::perms permissions);
void Rename(const std::filesystem::path& from, const std::filesystem::path& to);
// Atomic rename that fails instead of replacing; false when `to` exists.
bool RenameNoReplace(const std::filesystem::path& from, const std::filesystem::path& to);
void HardLink(const std::filesystem::path& from, const std::filesystem::path& to);
bool Remove(const std::filesystem::path& path) noexcept;
void SyncDirectory(const std::filesystem::path& directory);
void RandomBytes(std::span<std::byte> out);
std::uint64_t UnixNanos() noexcept;
std::uint64_t PageSize() noexcept;
std::int64_t ProcessId() noexcept;
}
}

316
interfaces/Crafter.DB-Format.cppm Normal file
View file

@ -0,0 +1,316 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// The on-disk format: byte-exact header, commit record, extent table, schema
// blob and the 16-byte variable-length handle. Every struct here is a POD
// that is read and written with memcpy; the static_asserts pin each offset so
// a reordered member cannot silently change the file format. Little-endian
// only (asserted), which is every target Crafter builds for.
export module Crafter.DB:Format;
import std;
export namespace Crafter::DB {
inline constexpr std::uint32_t pageSize = 4096;
inline constexpr std::uint16_t formatVersion = 1;
inline constexpr std::size_t maxExtents = 32;
inline constexpr std::size_t regionSlots = 4;
inline constexpr std::uint16_t regionCount = 2; // 0 rows, 1 heap; 2-3 reserved
inline constexpr std::uint32_t minExtentBytes = 16384;
inline constexpr std::uint16_t noColumn = 0xFFFF;
inline constexpr std::uint32_t inlineVarBytes = 12; // strings up to this length live in the handle
inline constexpr std::array<char, 8> fileMagic = { 'C', 'R', 'A', 'F', 'T', 'R', 'D', 'B' };
inline constexpr std::array<char, 4> schemaMagic = { 'S', 'C', 'H', 'M' };
inline constexpr std::uint32_t formatFlagLittleEndian = 1;
inline constexpr std::uint64_t commitFlagCleanClose = 1;
inline constexpr std::uint8_t rowFlagDeleted = 1;
static_assert(std::endian::native == std::endian::little, "Crafter.DB stores little-endian and does not byte-swap");
// Column kinds as stored in the descriptor table. Append only: values are
// part of the file format.
enum class Kind : std::uint8_t {
Flags = 0,
Bool = 1,
Int8 = 2,
Int16 = 3,
Int32 = 4,
Int64 = 5,
UInt8 = 6,
UInt16 = 7,
UInt32 = 8,
UInt64 = 9,
Float32 = 10,
Float64 = 11,
TimePoint = 12,
Duration = 13,
String = 14,
Blob = 15,
FixedBytes = 16,
};
inline constexpr std::uint8_t columnFlagOptional = 1; // payload of a std::optional; presenceOffset is valid
inline constexpr std::uint8_t columnFlagEnum = 2; // integer column backing an enum
inline constexpr std::uint8_t columnFlagPresence = 4; // the 1-byte presence column of a std::optional
constexpr std::string_view KindName(Kind kind) noexcept {
switch (kind) {
case Kind::Flags: return "flags";
case Kind::Bool: return "bool";
case Kind::Int8: return "i8";
case Kind::Int16: return "i16";
case Kind::Int32: return "i32";
case Kind::Int64: return "i64";
case Kind::UInt8: return "u8";
case Kind::UInt16: return "u16";
case Kind::UInt32: return "u32";
case Kind::UInt64: return "u64";
case Kind::Float32: return "f32";
case Kind::Float64: return "f64";
case Kind::TimePoint: return "timepoint";
case Kind::Duration: return "duration";
case Kind::String: return "string";
case Kind::Blob: return "blob";
case Kind::FixedBytes: return "bytes";
}
return "?";
}
struct Extent {
std::uint64_t fileOffset;
std::uint64_t length;
};
static_assert(sizeof(Extent) == 16);
// One of the two alternating commit slots. The writer fills the slot that
// is not current, so a torn write can only damage the slot whose CRC then
// fails, and the other slot still describes the previous commit.
struct CommitRecord {
std::uint64_t commitSeq;
std::uint64_t rowCount;
std::uint64_t liveCount;
std::uint64_t heapUsed;
std::uint64_t heapGarbage;
std::array<std::uint32_t, regionSlots> extentCount;
std::uint64_t fileLength;
std::uint64_t modifiedUnixNanos;
std::uint64_t flags;
std::array<std::uint8_t, 428> reserved;
std::uint32_t crc32c;
};
static_assert(sizeof(CommitRecord) == 512);
static_assert(__builtin_offsetof(CommitRecord, extentCount) == 40);
static_assert(__builtin_offsetof(CommitRecord, fileLength) == 56);
static_assert(__builtin_offsetof(CommitRecord, flags) == 72);
static_assert(__builtin_offsetof(CommitRecord, crc32c) == 508);
struct FileHeader {
std::array<char, 8> magic;
std::uint16_t formatVersion;
std::uint16_t headerPages;
std::uint32_t formatFlags;
std::uint32_t pageSize;
std::uint32_t rowSize;
std::uint16_t rowAlign;
std::uint16_t columnCount;
std::uint16_t orderedColumn;
std::uint16_t regionCount;
std::uint32_t schemaVersion;
std::uint32_t reserved0;
std::uint64_t schemaFingerprint;
std::uint32_t schemaBlobLength;
std::uint32_t minExtentBytes;
std::uint64_t createdUnixNanos;
std::array<std::uint8_t, 16> fileUuid;
std::uint32_t migratedFromVersion;
std::array<std::uint8_t, 44> reserved1;
std::array<std::array<Extent, maxExtents>, regionSlots> extents;
std::array<std::uint8_t, 896> reserved2;
std::array<CommitRecord, 2> commits;
};
static_assert(sizeof(FileHeader) == pageSize);
static_assert(__builtin_offsetof(FileHeader, schemaVersion) == 32);
static_assert(__builtin_offsetof(FileHeader, schemaFingerprint) == 40);
static_assert(__builtin_offsetof(FileHeader, createdUnixNanos) == 56);
static_assert(__builtin_offsetof(FileHeader, fileUuid) == 64);
static_assert(__builtin_offsetof(FileHeader, migratedFromVersion) == 80);
static_assert(__builtin_offsetof(FileHeader, extents) == 128);
static_assert(__builtin_offsetof(FileHeader, reserved2) == 2176);
static_assert(__builtin_offsetof(FileHeader, commits) == 3072);
struct SchemaBlobHeader {
std::array<char, 4> magic;
std::uint16_t schemaFormat;
std::uint16_t columnCount;
std::uint32_t rowSize;
std::uint32_t namePoolLength;
};
static_assert(sizeof(SchemaBlobHeader) == 16);
struct ColumnDescriptor {
std::uint16_t offset;
std::uint16_t size;
Kind kind;
std::uint8_t flags;
std::uint16_t presenceOffset;
std::uint16_t nameOffset;
std::uint16_t nameLength;
std::uint32_t paramNum; // chrono period numerator; element alignment for blobs; N for fixed bytes
std::uint32_t paramDen; // chrono period denominator; element size for blobs
std::uint32_t variant; // element kind for blobs; clock code for time points; 0 otherwise
};
static_assert(sizeof(ColumnDescriptor) == 24);
inline constexpr std::uint8_t maxKind = static_cast<std::uint8_t>(Kind::FixedBytes);
// The 16-byte handle of a variable-length column (Umbra / "German string").
// Bytes 4..16 are the first 12 bytes of the value, so a value of up to 12
// bytes is entirely inline and contiguous at (handle + 4). Longer values
// keep their 4-byte prefix inline and store an offset into the heap region
// where the whole value, prefix included, lives contiguously.
struct Var {
std::uint32_t length;
std::uint32_t prefix;
std::uint64_t tailOrOffset;
};
static_assert(sizeof(Var) == 16 && alignof(Var) == 8);
constexpr std::size_t KindSize(Kind kind) noexcept {
switch (kind) {
case Kind::Flags: case Kind::Bool: case Kind::Int8: case Kind::UInt8: return 1;
case Kind::Int16: case Kind::UInt16: return 2;
case Kind::Int32: case Kind::UInt32: case Kind::Float32: return 4;
case Kind::Int64: case Kind::UInt64: case Kind::Float64: case Kind::TimePoint: case Kind::Duration: return 8;
case Kind::String: case Kind::Blob: return sizeof(Var);
case Kind::FixedBytes: return 0; // size comes from the descriptor
}
return 0;
}
// FNV-1a 64. Chosen over std::hash because the result is part of the file
// format and must not change with the standard library.
inline constexpr std::uint64_t fnvOffset = 14695981039346656037ull;
inline constexpr std::uint64_t fnvPrime = 1099511628211ull;
constexpr std::uint64_t Fnv1a(std::uint64_t hash, std::string_view bytes) noexcept {
for (const char c : bytes) {
hash ^= static_cast<std::uint8_t>(c);
hash *= fnvPrime;
}
return hash;
}
constexpr std::uint64_t Fnv1a(std::uint64_t hash, std::uint64_t value) noexcept {
for (std::size_t i = 0; i < 8; i++) {
hash ^= (value >> (8 * i)) & 0xFF;
hash *= fnvPrime;
}
return hash;
}
// CRC-32C (Castagnoli), table driven. 508 bytes per commit is too little
// for the hardware instruction to matter.
namespace Detail {
consteval std::array<std::uint32_t, 256> MakeCrcTable() {
std::array<std::uint32_t, 256> table{};
for (std::uint32_t i = 0; i < 256; i++) {
std::uint32_t c = i;
for (std::int32_t k = 0; k < 8; k++) c = (c & 1) ? (0x82F63B78u ^ (c >> 1)) : (c >> 1);
table[i] = c;
}
return table;
}
inline constexpr std::array<std::uint32_t, 256> crcTable = MakeCrcTable();
}
constexpr std::uint32_t Crc32c(std::span<const std::byte> bytes) noexcept {
std::uint32_t crc = 0xFFFFFFFFu;
for (const std::byte b : bytes) crc = Detail::crcTable[(crc ^ static_cast<std::uint8_t>(b)) & 0xFF] ^ (crc >> 8);
return crc ^ 0xFFFFFFFFu;
}
inline std::uint32_t CommitCrc(const CommitRecord& record) noexcept {
return Crc32c(std::as_bytes(std::span(&record, 1)).first(__builtin_offsetof(CommitRecord, crc32c)));
}
inline bool CommitValid(const CommitRecord& record) noexcept {
return record.commitSeq != 0 && CommitCrc(record) == record.crc32c;
}
// A parsed schema blob: what the file says its columns are. Column 0 is
// always the flags byte at offset 0.
struct SchemaInfo {
std::vector<ColumnDescriptor> columns;
std::string namePool;
std::uint32_t rowSize = 0;
std::uint16_t rowAlign = 1;
std::string_view Name(std::size_t column) const noexcept {
const ColumnDescriptor& d = columns[column];
return std::string_view(namePool).substr(d.nameOffset, d.nameLength);
}
bool HasHeap() const noexcept {
return std::ranges::any_of(columns, [](const ColumnDescriptor& d) { return d.kind == Kind::String || d.kind == Kind::Blob; });
}
std::vector<std::byte> Serialize() const {
std::vector<std::byte> out(sizeof(SchemaBlobHeader) + columns.size() * sizeof(ColumnDescriptor) + namePool.size());
const SchemaBlobHeader header{ schemaMagic, 1, static_cast<std::uint16_t>(columns.size()), rowSize, static_cast<std::uint32_t>(namePool.size()) };
std::memcpy(out.data(), &header, sizeof(header));
std::memcpy(out.data() + sizeof(header), columns.data(), columns.size() * sizeof(ColumnDescriptor));
std::memcpy(out.data() + sizeof(header) + columns.size() * sizeof(ColumnDescriptor), namePool.data(), namePool.size());
return out;
}
// Returns false when the bytes are not a well-formed blob. Every field
// of every descriptor is range-checked here, because the engine and
// DynamicRow index rows with them without further checks.
static bool Parse(std::span<const std::byte> bytes, std::uint16_t rowAlign, SchemaInfo& out) {
if (bytes.size() < sizeof(SchemaBlobHeader)) return false;
SchemaBlobHeader header;
std::memcpy(&header, bytes.data(), sizeof(header));
if (header.magic != schemaMagic || header.schemaFormat != 1 || header.columnCount == 0 || header.rowSize == 0) return false;
const std::size_t descriptorBytes = std::size_t(header.columnCount) * sizeof(ColumnDescriptor);
if (bytes.size() < sizeof(header) + descriptorBytes + header.namePoolLength) return false;
out.columns.resize(header.columnCount);
std::memcpy(out.columns.data(), bytes.data() + sizeof(header), descriptorBytes);
out.namePool.assign(reinterpret_cast<const char*>(bytes.data() + sizeof(header) + descriptorBytes), header.namePoolLength);
out.rowSize = header.rowSize;
out.rowAlign = rowAlign;
for (std::size_t i = 0; i < out.columns.size(); i++) {
const ColumnDescriptor& d = out.columns[i];
if (static_cast<std::uint8_t>(d.kind) > maxKind) return false;
if (std::size_t(d.nameOffset) + d.nameLength > out.namePool.size()) return false;
if (d.size == 0 || std::size_t(d.offset) + d.size > out.rowSize) return false;
if (d.kind != Kind::FixedBytes && KindSize(d.kind) != d.size) return false;
if (d.presenceOffset != noColumn && d.presenceOffset >= out.rowSize) return false;
if ((d.flags & columnFlagOptional) != 0 && d.presenceOffset == noColumn) return false;
if (i == 0 && (d.kind != Kind::Flags || d.offset != 0)) return false;
if (i != 0 && d.kind == Kind::Flags) return false;
}
return true;
}
// Canonical hash of the layout; the same function runs at compile time
// on Layout<T> so the two sides agree byte for byte. The ordered
// column is part of it: changing `orderedBy` changes what the file
// promises about its rows.
std::uint64_t Fingerprint(std::uint16_t orderedColumn) const noexcept {
std::uint64_t hash = Fnv1a(fnvOffset, "crafter-db-v1");
hash = Fnv1a(hash, rowSize);
hash = Fnv1a(hash, rowAlign);
hash = Fnv1a(hash, orderedColumn);
for (std::size_t i = 0; i < columns.size(); i++) {
const ColumnDescriptor& d = columns[i];
hash = Fnv1a(hash, Name(i));
hash = Fnv1a(hash, static_cast<std::uint64_t>(d.kind));
hash = Fnv1a(hash, d.offset);
hash = Fnv1a(hash, d.size);
hash = Fnv1a(hash, d.flags);
hash = Fnv1a(hash, d.paramNum);
hash = Fnv1a(hash, d.paramDen);
hash = Fnv1a(hash, d.variant);
}
return hash;
}
};
}

372
interfaces/Crafter.DB-Formatter.cppm Normal file
View file

@ -0,0 +1,372 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// The customisation point for member types, in the spirit of std::formatter:
// specialise Crafter::DB::Formatter<T> to say how a T is stored. Fixed-width
// formatters map T to a storable `Stored` type; variable-length formatters
// map it to the 16-byte Var handle and write the bytes into the heap. This
// partition also holds the reader-side handle views (StringRef, BlobRef), the
// pre-encoded search Key, and the built-in formatters for std::string,
// std::vector<Trivial> and std::chrono types.
export module Crafter.DB:Formatter;
import std;
import :Format;
import :Error;
import :Reflect;
export namespace Crafter::DB {
template<class T>
concept Scalar = (std::is_arithmetic_v<T> && !std::same_as<T, long double>) || std::is_enum_v<T> || std::same_as<T, std::byte>;
template<class T>
concept ByteLike = std::same_as<T, char> || std::same_as<T, unsigned char> || std::same_as<T, signed char> || std::same_as<T, std::byte> || std::same_as<T, char8_t>;
// The column kind of a scalar. `char` is Int8 whatever the platform's
// signedness, so a file's fingerprint does not depend on the ABI.
template<Scalar T>
consteval Kind ScalarKind() {
if constexpr (std::same_as<T, bool>) return Kind::Bool;
else if constexpr (std::is_enum_v<T>) return ScalarKind<std::underlying_type_t<T>>();
else if constexpr (std::same_as<T, std::byte>) return Kind::UInt8;
else if constexpr (std::same_as<T, char>) return Kind::Int8;
else if constexpr (std::same_as<T, wchar_t>) return sizeof(wchar_t) == 2 ? Kind::UInt16 : Kind::Int32;
else if constexpr (std::is_floating_point_v<T>) return sizeof(T) == 4 ? Kind::Float32 : Kind::Float64;
else if constexpr (std::is_signed_v<T>) return sizeof(T) == 1 ? Kind::Int8 : sizeof(T) == 2 ? Kind::Int16 : sizeof(T) == 4 ? Kind::Int32 : Kind::Int64;
else return sizeof(T) == 1 ? Kind::UInt8 : sizeof(T) == 2 ? Kind::UInt16 : sizeof(T) == 4 ? Kind::UInt32 : Kind::UInt64;
}
// A 32-bit code that identifies a type in the schema: the scalar kind
// for scalars, a hash of the compiler's spelling otherwise.
template<class T>
consteval std::uint32_t TypeCode() {
if constexpr (Scalar<T>) return static_cast<std::uint32_t>(ScalarKind<T>());
else return static_cast<std::uint32_t>(Fnv1a(fnvOffset, TypeName<T>()) & 0xFFFFFFFF) | 0x80000000u;
}
// Reader-side view of a table's heap region. `used` is the published
// length, so a handle pointing past it (a corrupt file) is caught before
// the bytes are touched.
struct Heap {
const std::byte* base = nullptr;
const std::atomic<std::uint64_t>* used = nullptr;
std::span<const std::byte> Read(std::uint64_t offset, std::uint64_t length) const {
const std::uint64_t limit = used->load(std::memory_order_acquire);
if (offset > limit || length > limit - offset) throw CorruptError(std::format("heap reference [{}, +{}) is outside the committed heap of {} bytes", offset, length, limit));
return { base + offset, static_cast<std::size_t>(length) };
}
};
// Writer-side cursor into the heap region. Formatters append through it;
// the table decides when the bytes become visible. Grow() is only reached
// when an extent fills up, so the virtual call never sits on the fast path.
class HeapWriter {
public:
virtual ~HeapWriter() = default;
std::uint64_t Write(std::span<const std::byte> bytes, std::uint32_t align = 1) {
std::uint64_t at = cursor;
if (align > 1) at = (at + align - 1) & ~std::uint64_t(align - 1);
if (at + bytes.size() > capacity) Grow(at + bytes.size());
if (!bytes.empty()) std::memcpy(base + at, bytes.data(), bytes.size());
cursor = at + bytes.size();
return at;
}
std::uint64_t Cursor() const noexcept { return cursor; }
void Rewind(std::uint64_t to) noexcept { cursor = to; }
protected:
virtual void Grow(std::uint64_t neededCapacity) = 0;
std::byte* base = nullptr;
std::uint64_t cursor = 0;
std::uint64_t capacity = 0;
};
// Whether a value of this length and element alignment is stored inside
// the handle. Alignments above 4 cannot be inline because the inline bytes
// start at handle offset 4.
constexpr bool VarInline(std::uint64_t length, std::uint32_t align) noexcept {
return length <= inlineVarBytes && align <= 4;
}
inline Var EncodeVar(std::span<const std::byte> bytes, HeapWriter& heap, std::uint32_t align = 1) {
if (bytes.size() >= std::uint64_t(1) << 32) throw Error(std::format("a value of {} bytes does not fit a Crafter.DB column (limit 4 GiB)", bytes.size()));
Var var{};
var.length = static_cast<std::uint32_t>(bytes.size());
if (VarInline(bytes.size(), align)) {
if (!bytes.empty()) std::memcpy(reinterpret_cast<std::byte*>(&var) + 4, bytes.data(), bytes.size());
} else {
// An empty blob can land here when its element alignment forbids
// inline storage; it has no prefix bytes to copy.
if (!bytes.empty()) std::memcpy(&var.prefix, bytes.data(), std::min<std::size_t>(bytes.size(), 4));
var.tailOrOffset = heap.Write(bytes, align);
}
return var;
}
// The bytes a handle refers to, inline or in the heap.
template<std::uint32_t Align = 1>
std::span<const std::byte> VarBytes(const std::byte* handle, const Heap& heap) {
Var var;
std::memcpy(&var, handle, sizeof(var));
if (VarInline(var.length, Align)) return { handle + 4, var.length };
return heap.Read(var.tailOrOffset, var.length);
}
// A search key encoded once, so a scan compares one 8-byte word per row.
// `head` is the first 8 bytes of a matching handle (length | prefix << 32)
// and `tail` the second 8 bytes when the value is inline.
class Key {
public:
explicit Key(std::string_view value) : text(value) {
head = static_cast<std::uint64_t>(static_cast<std::uint32_t>(text.size()));
std::uint32_t prefix = 0;
if (!text.empty()) std::memcpy(&prefix, text.data(), std::min<std::size_t>(text.size(), 4));
head |= std::uint64_t(prefix) << 32;
tail = 0;
if (text.size() > 4 && text.size() <= inlineVarBytes) std::memcpy(&tail, text.data() + 4, text.size() - 4);
}
std::uint64_t head;
std::uint64_t tail;
std::string text;
};
// Handle view of a string column. Points into the mapped row (and, for
// long values, the mapped heap); valid for as long as the table and that
// row are. Converts to std::string_view on demand; its comparison operators
// avoid that conversion and reject almost every non-match on the first
// 8-byte compare.
class StringRef {
public:
StringRef() = default;
StringRef(const std::byte* handle, const Heap* heap) noexcept : handle(handle), heap(heap) {}
std::uint32_t Length() const noexcept {
std::uint32_t length;
std::memcpy(&length, handle, sizeof(length));
return length;
}
bool Empty() const noexcept { return Length() == 0; }
std::string_view View() const {
const std::span<const std::byte> bytes = VarBytes<1>(handle, *heap);
return { reinterpret_cast<const char*>(bytes.data()), bytes.size() };
}
operator std::string_view() const { return View(); }
std::uint64_t Head() const noexcept {
std::uint64_t head;
std::memcpy(&head, handle, sizeof(head));
return head;
}
std::uint64_t Tail() const noexcept {
std::uint64_t tail;
std::memcpy(&tail, handle + 8, sizeof(tail));
return tail;
}
friend bool operator==(StringRef ref, const Key& key) {
if (ref.Head() != key.head) return false;
if (key.text.size() <= inlineVarBytes) return ref.Tail() == key.tail;
const std::span<const std::byte> bytes = ref.heap->Read(ref.Tail(), key.text.size());
return std::memcmp(bytes.data() + 4, key.text.data() + 4, key.text.size() - 4) == 0;
}
friend bool operator==(StringRef ref, std::string_view value) {
std::uint64_t head = static_cast<std::uint64_t>(static_cast<std::uint32_t>(value.size()));
std::uint32_t prefix = 0;
if (!value.empty()) std::memcpy(&prefix, value.data(), std::min<std::size_t>(value.size(), 4));
head |= std::uint64_t(prefix) << 32;
if (ref.Head() != head) return false;
if (value.size() <= inlineVarBytes) {
std::uint64_t tail = 0;
if (value.size() > 4) std::memcpy(&tail, value.data() + 4, value.size() - 4);
return ref.Tail() == tail;
}
const std::span<const std::byte> bytes = ref.heap->Read(ref.Tail(), value.size());
return std::memcmp(bytes.data() + 4, value.data() + 4, value.size() - 4) == 0;
}
friend bool operator==(StringRef a, StringRef b) {
if (a.Head() != b.Head()) return false;
if (a.Length() <= inlineVarBytes) return a.Tail() == b.Tail();
return a.View() == b.View();
}
friend std::strong_ordering operator<=>(StringRef ref, std::string_view value) {
const std::int32_t c = ref.View().compare(value);
return c < 0 ? std::strong_ordering::less : c > 0 ? std::strong_ordering::greater : std::strong_ordering::equal;
}
friend std::strong_ordering operator<=>(StringRef a, StringRef b) { return a <=> b.View(); }
private:
const std::byte* handle = nullptr;
const Heap* heap = nullptr;
};
// Handle view of a blob column holding elements of E.
template<class E>
class BlobRef {
public:
BlobRef() = default;
BlobRef(const std::byte* handle, const Heap* heap) noexcept : handle(handle), heap(heap) {}
std::size_t Size() const noexcept {
std::uint32_t length;
std::memcpy(&length, handle, sizeof(length));
return length / sizeof(E);
}
std::span<const E> View() const {
const std::span<const std::byte> bytes = VarBytes<alignof(E)>(handle, *heap);
return { reinterpret_cast<const E*>(bytes.data()), bytes.size() / sizeof(E) };
}
operator std::span<const E>() const { return View(); }
friend bool operator==(BlobRef ref, std::span<const E> value) {
const std::span<const E> mine = ref.View();
return mine.size() == value.size() && std::memcmp(mine.data(), value.data(), value.size_bytes()) == 0;
}
private:
const std::byte* handle = nullptr;
const Heap* heap = nullptr;
};
// Specialise this for your own member types. Left undefined on purpose:
// "no Formatter" is then a clean concept failure with a readable message.
template<class T>
struct Formatter;
// A fixed-width formatter maps T to a `Stored` type that is itself storable
// (scalar, enum, std::array, aggregate of storables). Optional members:
// `kind` (overrides the column kind), `paramNum`/`paramDen` (chrono period).
template<class T>
concept FixedFormatter = requires { typename Formatter<T>::Stored; }
&& !std::same_as<typename Formatter<T>::Stored, Var>
&& requires(const T& value, const typename Formatter<T>::Stored& stored) {
{ Formatter<T>::Encode(value) } -> std::same_as<typename Formatter<T>::Stored>;
{ Formatter<T>::Decode(stored) } -> std::convertible_to<T>;
};
// A variable-length formatter maps T to a Var handle. `View` is what a Row
// hands out (zero-copy); Materialize(View) -> T is optional and defaults to
// T(view). Optional: `kind` (String or Blob), `align` for blob elements.
template<class T>
concept VarFormatter = requires { typename Formatter<T>::Stored; typename Formatter<T>::View; }
&& std::same_as<typename Formatter<T>::Stored, Var>
&& requires(const T& value, HeapWriter& writer, const std::byte* handle, const Heap& heap) {
{ Formatter<T>::Encode(value, writer) } -> std::same_as<Var>;
{ Formatter<T>::Decode(handle, heap) } -> std::same_as<typename Formatter<T>::View>;
};
template<class T>
concept HasFormatterKind = requires { { Formatter<T>::kind } -> std::convertible_to<Kind>; };
template<class T>
concept HasFormatterPeriod = requires { { Formatter<T>::paramNum } -> std::convertible_to<std::uint32_t>; { Formatter<T>::paramDen } -> std::convertible_to<std::uint32_t>; };
template<class T>
concept HasFormatterVariant = requires { { Formatter<T>::variant } -> std::convertible_to<std::uint32_t>; };
template<class T>
concept HasMaterialize = VarFormatter<T> && requires(typename Formatter<T>::View view) { { Formatter<T>::Materialize(view) } -> std::same_as<T>; };
template<VarFormatter T>
T MaterializeVar(typename Formatter<T>::View view) {
if constexpr (HasMaterialize<T>) return Formatter<T>::Materialize(view);
else return T(view);
}
template<>
struct Formatter<std::string> {
using Stored = Var;
using View = StringRef;
static constexpr Kind kind = Kind::String;
static Var Encode(std::string_view value, HeapWriter& heap) { return EncodeVar(std::as_bytes(std::span(value)), heap, 1); }
static StringRef Decode(const std::byte* handle, const Heap& heap) noexcept { return { handle, &heap }; }
static std::string Materialize(StringRef view) { return std::string(view.View()); }
};
template<class E> requires (std::is_trivially_copyable_v<E> && !std::is_pointer_v<E> && !std::is_member_pointer_v<E>)
struct Formatter<std::vector<E>> {
using Stored = Var;
using View = BlobRef<E>;
static constexpr Kind kind = Kind::Blob;
static constexpr std::uint32_t paramNum = alignof(E);
static constexpr std::uint32_t paramDen = sizeof(E);
static constexpr std::uint32_t variant = TypeCode<E>(); // the element type is part of the schema
static Var Encode(std::span<const E> value, HeapWriter& heap) { return EncodeVar(std::as_bytes(value), heap, alignof(E)); }
static BlobRef<E> Decode(const std::byte* handle, const Heap& heap) noexcept { return { handle, &heap }; }
static std::vector<E> Materialize(BlobRef<E> view) {
const std::span<const E> elements = view.View();
return std::vector<E>(elements.begin(), elements.end());
}
};
template<class Rep, class Period>
struct Formatter<std::chrono::duration<Rep, Period>> {
static_assert(Period::num > 0 && Period::num <= 0xFFFFFFFF && Period::den > 0 && Period::den <= 0xFFFFFFFF, "chrono period does not fit the column descriptor");
using Stored = Rep;
static constexpr Kind kind = Kind::Duration;
static constexpr std::uint32_t paramNum = static_cast<std::uint32_t>(Period::num);
static constexpr std::uint32_t paramDen = static_cast<std::uint32_t>(Period::den);
static constexpr Rep Encode(const std::chrono::duration<Rep, Period>& value) noexcept { return value.count(); }
static constexpr std::chrono::duration<Rep, Period> Decode(const Rep& stored) noexcept { return std::chrono::duration<Rep, Period>(stored); }
};
template<class Clock, class Duration>
struct Formatter<std::chrono::time_point<Clock, Duration>> {
using Period = typename Duration::period;
static_assert(Period::num > 0 && Period::num <= 0xFFFFFFFF && Period::den > 0 && Period::den <= 0xFFFFFFFF, "chrono period does not fit the column descriptor");
using Stored = typename Duration::rep;
static constexpr Kind kind = Kind::TimePoint;
static constexpr std::uint32_t paramNum = static_cast<std::uint32_t>(Period::num);
static constexpr std::uint32_t paramDen = static_cast<std::uint32_t>(Period::den);
static constexpr std::uint32_t variant = TypeCode<Clock>(); // which clock the epoch belongs to
static constexpr Stored Encode(const std::chrono::time_point<Clock, Duration>& value) noexcept { return value.time_since_epoch().count(); }
static constexpr std::chrono::time_point<Clock, Duration> Decode(const Stored& stored) noexcept { return std::chrono::time_point<Clock, Duration>(Duration(stored)); }
};
// An inline, fixed-capacity string column: N bytes in the row, NUL padded,
// no heap. For codes with a known maximum length (BIC, currency, ISIN).
template<std::size_t N>
struct FixedString {
std::array<char, N> chars{};
constexpr FixedString() = default;
constexpr FixedString(std::string_view value) {
if (value.size() > N) throw std::length_error(std::format("'{}' does not fit a FixedString<{}>", value, N));
std::ranges::copy(value, chars.begin());
}
constexpr std::string_view View() const noexcept {
const auto end = std::ranges::find(chars, '\0');
return { chars.data(), static_cast<std::size_t>(end - chars.begin()) };
}
constexpr operator std::string_view() const noexcept { return View(); }
friend constexpr bool operator==(const FixedString&, const FixedString&) = default;
friend constexpr auto operator<=>(const FixedString&, const FixedString&) = default;
friend constexpr bool operator==(const FixedString& a, std::string_view b) noexcept { return a.View() == b; }
};
static_assert(VarFormatter<std::string>);
static_assert(VarFormatter<std::vector<std::byte>>);
static_assert(VarFormatter<std::vector<std::uint32_t>>);
static_assert(FixedFormatter<std::chrono::system_clock::time_point>);
static_assert(FixedFormatter<std::chrono::milliseconds>);
static_assert(HasMaterialize<std::string>);
}

212
interfaces/Crafter.DB-Migration.cppm Normal file
View file

@ -0,0 +1,212 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// Schema migrations in C++. A table struct names its predecessor with
// `using Previous = OldStruct;` and either supplies
// `static T Migrate(OldStruct&&)` or, for purely additive changes, nothing:
// members are then copied by name and new ones value-initialised. Opening a
// file whose version is older than the struct's walks the chain to the type
// with that version and streams every live row through the conversions into
// a new file that replaces the old one atomically.
export module Crafter.DB:Migration;
import std;
import :Error;
import :Reflect;
import :Format;
import :Formatter;
import :Schema;
import :File;
import :Storage;
import :Codec;
export namespace Crafter::DB {
template<class From, std::size_t I, class To>
consteval std::size_t FindMemberByName() {
constexpr std::string_view name = MemberName<I, From>();
std::size_t found = noMember;
[&]<std::size_t... J>(std::index_sequence<J...>) {
((void)[&] { if (MemberName<J, To>() == name) found = J; }(), ...);
}(std::make_index_sequence<memberCount<To>>{});
return found;
}
// Additive migration: every member of From must exist in To with the same
// name and type; members only in To are value-initialised.
template<Aggregate To, Aggregate From>
To CopyByName(From&& from) {
To to{};
[&]<std::size_t... I>(std::index_sequence<I...>) {
((void)[&] {
constexpr std::size_t j = FindMemberByName<From, I, To>();
static_assert(j != noMember, Message({ "Crafter.DB: member '", MemberName<I, From>(), "' of ", TypeName<From>(), " has no member of the same name in ", TypeName<To>(), "; a removed or renamed column needs an explicit `static ", TypeName<To>(), " Migrate(", TypeName<From>(), "&&)`." }));
if constexpr (j != noMember) {
static_assert(std::is_same_v<MemberType<I, From>, MemberType<j, To>>, Message({ "Crafter.DB: member '", MemberName<I, From>(), "' changed type between ", TypeName<From>(), " and ", TypeName<To>(), "; a type change needs an explicit `static ", TypeName<To>(), " Migrate(", TypeName<From>(), "&&)`." }));
Member<j>(to) = std::move(Member<I>(from));
}
}(), ...);
}(std::make_index_sequence<memberCount<From>>{});
return to;
}
// One step up the chain: Previous -> Next.
template<class Next>
Next ConvertStep(typename Next::Previous&& from) {
if constexpr (HasMigrate<Next>) return Next::Migrate(std::move(from));
else return CopyByName<Next>(std::move(from));
}
// The type directly above `From` in To's chain.
template<class To, class From>
consteval auto NextAbove() {
if constexpr (std::is_same_v<typename To::Previous, From>) return std::type_identity<To>{};
else return NextAbove<typename To::Previous, From>();
}
template<class To, class From>
To ConvertChain(From&& from) {
if constexpr (std::is_same_v<From, To>) return std::move(from);
else {
using Next = typename decltype(NextAbove<To, From>())::type;
Next next = ConvertStep<Next>(std::move(from));
return ConvertChain<To, Next>(std::move(next));
}
}
std::string DescribeColumns(const SchemaInfo& schema) {
std::string out = "[";
for (std::size_t i = 1; i < schema.columns.size(); i++) {
if (i != 1) out += ", ";
out += std::format("{}:{}", schema.Name(i), KindName(schema.columns[i].kind));
}
return out + "]";
}
template<Aggregate T>
void RequireFingerprint(const std::filesystem::path& path, const FileHeader& header, const SchemaInfo& schema) {
if (header.schemaFingerprint == Layout<T>::fingerprint && header.rowSize == Layout<T>::rowSize && header.orderedColumn == Layout<T>::orderedColumn) return;
const std::string onDisk = DescribeColumns(schema);
const std::string compiled = DescribeColumns(MakeSchemaInfo<T>());
if (onDisk == compiled && header.orderedColumn != Layout<T>::orderedColumn) {
const auto name = [&](std::uint16_t column) { return column == noColumn ? std::string("none") : std::string(schema.Name(column)); };
throw SchemaError(std::format("{}: {} version {} on disk is ordered by {} but the compiled struct is ordered by {}: `orderedBy` changed without bumping `version`", path.string(), TypeName<T>(), header.schemaVersion, name(header.orderedColumn), name(Layout<T>::orderedColumn)));
}
throw SchemaError(std::format("{}: {} version {} on disk has columns {} but the compiled struct has {}: the struct changed without bumping `version`", path.string(), TypeName<T>(), header.schemaVersion, onDisk, compiled));
}
// Writes the converted live rows of `old` into a fresh file and returns
// its path. Streaming keeps memory flat; `sorted` collects everything and
// stable-sorts by the new orderedBy first, for the case where the old data
// is not in the order the new struct declares.
template<TableType T, Aggregate Old>
std::filesystem::path WriteConverted(const TableFile& old, const std::filesystem::path& path, const OpenOptions& options, bool sorted) {
const TableStats stats = old.Stats();
CreateParams params;
params.schema = MakeSchemaInfo<T>();
params.schemaVersion = Layout<T>::version;
params.fingerprint = Layout<T>::fingerprint;
params.orderedColumn = Layout<T>::orderedColumn;
params.initialRowBytes = stats.liveCount * Layout<T>::rowSize;
params.initialHeapBytes = stats.heapUsed - std::min(stats.heapUsed, stats.heapGarbage);
params.migratedFromVersion = old.Header().schemaVersion;
OpenOptions targetOptions = options;
targetOptions.readOnly = false;
targetOptions.createDirectories = false;
const std::filesystem::path temp = TableFile::TempPath(path);
TableFile::Create(temp, params, targetOptions);
try {
TableFile target(temp, targetOptions);
const std::uint64_t slots = old.SlotCount();
if (sorted) {
std::vector<T> rows;
rows.reserve(stats.liveCount);
for (std::uint64_t i = 0; i < slots; i++) {
const std::byte* row = old.Slot(i);
if (!TableFile::Deleted(row)) rows.push_back(ConvertChain<T, Old>(DecodeRow<Old>(row, old.HeapView())));
}
if constexpr (HasOrderedBy<T>) std::ranges::stable_sort(rows, {}, [](const T& row) { return row.*T::orderedBy; });
AppendRows<T>(target, std::span<const T>(rows));
} else {
for (std::uint64_t i = 0; i < slots; i++) {
const std::byte* row = old.Slot(i);
if (TableFile::Deleted(row)) continue;
T converted = ConvertChain<T, Old>(DecodeRow<Old>(row, old.HeapView()));
AppendRows<T>(target, std::span<const T>(&converted, 1));
}
}
target.Flush(true);
} catch (...) {
Os::Remove(temp);
throw;
}
return temp;
}
template<TableType T, Aggregate Old>
void RewriteFrom(const std::filesystem::path& path, const OpenOptions& options) {
OpenOptions sourceOptions = options;
sourceOptions.readOnly = false;
sourceOptions.mode = OpenMode::OpenExisting;
TableFile old(path, sourceOptions);
const std::uint32_t oldVersion = old.Header().schemaVersion;
std::filesystem::path temp;
try {
temp = WriteConverted<T, Old>(old, path, options, false);
} catch (const OrderError&) {
if constexpr (!HasOrderedBy<T>) throw;
temp = WriteConverted<T, Old>(old, path, options, true);
}
try {
if (options.backupBeforeMigrate) {
const std::filesystem::path backup = TableFile::BackupPath(path, oldVersion);
Os::Remove(backup);
Os::HardLink(path, backup);
}
old.Close();
TableFile::Replace(path, temp);
} catch (...) {
Os::Remove(temp);
throw;
}
}
template<TableType T, Aggregate Old>
bool WalkChain(const std::filesystem::path& path, const Inspection& info, const OpenOptions& options) {
if (Layout<Old>::version == info.header.schemaVersion) {
RequireFingerprint<Old>(path, info.header, info.schema);
RewriteFrom<T, Old>(path, options);
return true;
}
if constexpr (HasPrevious<Old>) return WalkChain<T, typename Old::Previous>(path, info, options);
else return false;
}
template<class T>
consteval std::uint32_t OldestVersion() {
if constexpr (HasPrevious<T>) return OldestVersion<typename T::Previous>();
else return Layout<T>::version;
}
// Brings a file whose version differs from T's up to date, or throws.
template<TableType T>
void MigrateTable(const std::filesystem::path& path, const Inspection& info, const OpenOptions& options) {
const std::uint32_t fileVersion = info.header.schemaVersion;
if (fileVersion == Layout<T>::version) return;
if (fileVersion > Layout<T>::version) throw SchemaError(std::format("{}: file is at version {} but this build of {} is version {}; the file was written by a newer build", path.string(), fileVersion, TypeName<T>(), Layout<T>::version));
if (options.readOnly) throw MigrationError(std::format("{}: file is at version {} and {} is version {}, but the table was opened read-only", path.string(), fileVersion, TypeName<T>(), Layout<T>::version));
if (!options.migrate) throw MigrationError(std::format("{}: file is at version {} and {} is version {}, and migration was disabled", path.string(), fileVersion, TypeName<T>(), Layout<T>::version));
if (!WalkChain<T, T>(path, info, options)) {
throw SchemaError(std::format("{}: no migration path from on-disk version {} to {}: the Previous chain of {} stops at version {}", path.string(), fileVersion, Layout<T>::version, TypeName<T>(), OldestVersion<T>()));
}
}
// Explicit form: migrate the file at `path` to T without keeping it open.
template<TableType T>
void Migrate(const std::filesystem::path& path, const OpenOptions& options = {}) {
MigrateTable<T>(path, TableFile::Inspect(path), options);
}
std::uint32_t PeekVersion(const std::filesystem::path& path) {
return TableFile::Inspect(path).header.schemaVersion;
}
}

205
interfaces/Crafter.DB-Query.cppm Normal file
View file

@ -0,0 +1,205 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// Queries are C++. A lambda over Row<T> is the general form; column tags
// give a readable expression form (iban == x && amount > 100) whose types
// the engine can look into: comparisons on the orderedBy column found in a
// top-level && chain become binary-search bounds instead of a scan, and
// string comparisons pre-encode their Key once instead of once per row.
export module Crafter.DB:Query;
import std;
import :Reflect;
import :Formatter;
import :Schema;
import :Storage;
import :Codec;
import :Row;
export namespace Crafter::DB {
enum class CompareOp : std::uint8_t { Equal, NotEqual, Less, LessEqual, Greater, GreaterEqual };
template<auto Ptr, CompareOp Op, class Operand>
struct Compare {
using Table = MemberClass<Ptr>;
static constexpr auto pointer = Ptr;
static constexpr CompareOp op = Op;
Operand operand;
bool operator()(Row<Table> row) const {
const auto value = row.template Get<Ptr>();
if constexpr (Op == CompareOp::Equal) return value == operand;
else if constexpr (Op == CompareOp::NotEqual) return !(value == operand);
else if constexpr (Op == CompareOp::Less) return value < operand;
else if constexpr (Op == CompareOp::LessEqual) return value <= operand;
else if constexpr (Op == CompareOp::Greater) return value > operand;
else return value >= operand;
}
};
template<class A, class B>
struct And {
A a;
B b;
template<class R> bool operator()(R row) const { return a(row) && b(row); }
};
template<class A, class B>
struct Or {
A a;
B b;
template<class R> bool operator()(R row) const { return a(row) || b(row); }
};
template<class A>
struct Not {
A a;
template<class R> bool operator()(R row) const { return !a(row); }
};
template<class P> struct IsExpression : std::false_type {};
template<auto Ptr, CompareOp Op, class V> struct IsExpression<Compare<Ptr, Op, V>> : std::true_type {};
template<class A, class B> struct IsExpression<And<A, B>> : std::true_type {};
template<class A, class B> struct IsExpression<Or<A, B>> : std::true_type {};
template<class A> struct IsExpression<Not<A>> : std::true_type {};
template<class P>
concept Expression = IsExpression<std::remove_cvref_t<P>>::value;
template<class M>
concept StringColumn = std::is_same_v<M, std::string>;
template<auto Ptr, CompareOp Op, class V>
auto MakeCompare(V&& value) {
using M = MemberTypeOf<Ptr>;
if constexpr (StringColumn<M>) {
if constexpr (Op == CompareOp::Equal || Op == CompareOp::NotEqual) return Compare<Ptr, Op, Key>{ Key(std::string_view(value)) };
else return Compare<Ptr, Op, std::string>{ std::string(std::string_view(value)) };
} else {
return Compare<Ptr, Op, M>{ static_cast<M>(std::forward<V>(value)) };
}
}
template<auto Ptr, class V> auto operator==(Column<Ptr>, V&& value) { return MakeCompare<Ptr, CompareOp::Equal>(std::forward<V>(value)); }
template<auto Ptr, class V> auto operator!=(Column<Ptr>, V&& value) { return MakeCompare<Ptr, CompareOp::NotEqual>(std::forward<V>(value)); }
template<auto Ptr, class V> auto operator<(Column<Ptr>, V&& value) { return MakeCompare<Ptr, CompareOp::Less>(std::forward<V>(value)); }
template<auto Ptr, class V> auto operator<=(Column<Ptr>, V&& value) { return MakeCompare<Ptr, CompareOp::LessEqual>(std::forward<V>(value)); }
template<auto Ptr, class V> auto operator>(Column<Ptr>, V&& value) { return MakeCompare<Ptr, CompareOp::Greater>(std::forward<V>(value)); }
template<auto Ptr, class V> auto operator>=(Column<Ptr>, V&& value) { return MakeCompare<Ptr, CompareOp::GreaterEqual>(std::forward<V>(value)); }
template<Expression A, Expression B> auto operator&&(A a, B b) { return And<A, B>{ std::move(a), std::move(b) }; }
template<Expression A, Expression B> auto operator||(A a, B b) { return Or<A, B>{ std::move(a), std::move(b) }; }
template<Expression A> auto operator!(A a) { return Not<A>{ std::move(a) }; }
// Bounds on the ordered column harvested from a predicate's top-level
// conjunction. Anything under || or ! contributes nothing (the whole
// predicate still runs as the filter, so this is only ever a narrowing).
template<class K>
struct Bounds {
std::optional<K> lower;
bool lowerExclusive = false;
std::optional<K> upper;
bool upperExclusive = false;
void Lower(const K& key, bool exclusive) {
if (!lower || *lower < key || (*lower == key && exclusive)) {
lower = key;
lowerExclusive = exclusive;
}
}
void Upper(const K& key, bool exclusive) {
if (!upper || key < *upper || (*upper == key && exclusive)) {
upper = key;
upperExclusive = exclusive;
}
}
};
template<class P> struct IsAndType : std::false_type {};
template<class A, class B> struct IsAndType<And<A, B>> : std::true_type {};
template<class P> struct IsCompareType : std::false_type {};
template<auto Ptr, CompareOp Op, class V> struct IsCompareType<Compare<Ptr, Op, V>> : std::true_type {};
template<class T, class P> requires HasOrderedBy<T>
void CollectBounds(const P& predicate, Bounds<OrderedKey<T>>& bounds) {
if constexpr (IsAndType<P>::value) {
CollectBounds<T>(predicate.a, bounds);
CollectBounds<T>(predicate.b, bounds);
} else if constexpr (IsCompareType<P>::value) {
if constexpr (SameMemberPointer<P::pointer, T::orderedBy>()) {
const OrderedKey<T> key = static_cast<OrderedKey<T>>(predicate.operand);
switch (P::op) {
case CompareOp::Equal: bounds.Lower(key, false); bounds.Upper(key, false); break;
case CompareOp::Less: bounds.Upper(key, true); break;
case CompareOp::LessEqual: bounds.Upper(key, false); break;
case CompareOp::Greater: bounds.Lower(key, true); break;
case CompareOp::GreaterEqual: bounds.Lower(key, false); break;
case CompareOp::NotEqual: break;
}
}
}
}
// A lazy range of Row<T> with the conveniences a result needs. Chainable:
// table.Range<...>(a, b).Where(pred).Select<&T::amount>().
template<TableType T, std::ranges::view R> requires std::is_same_v<std::ranges::range_value_t<R>, Row<T>>
class Query : public std::ranges::view_interface<Query<T, R>> {
public:
Query() = default;
explicit Query(R range) : range(std::move(range)) {}
// A Query is iterable as a const object whenever its underlying view
// is (LiveRange is; std::ranges::filter_view, by the standard's
// design, is not).
template<class Self> requires std::ranges::range<std::conditional_t<std::is_const_v<std::remove_reference_t<Self>>, const R, R>>
auto begin(this Self&& self) { return std::ranges::begin(self.range); }
template<class Self> requires std::ranges::range<std::conditional_t<std::is_const_v<std::remove_reference_t<Self>>, const R, R>>
auto end(this Self&& self) { return std::ranges::end(self.range); }
template<class Self> requires std::ranges::range<std::conditional_t<std::is_const_v<std::remove_reference_t<Self>>, const R, R>>
std::vector<T> Materialize(this Self&& self) {
std::vector<T> out;
for (Row<T> row : self.range) out.push_back(row.Materialize());
return out;
}
template<class Self> requires std::ranges::range<std::conditional_t<std::is_const_v<std::remove_reference_t<Self>>, const R, R>>
std::uint64_t Count(this Self&& self) {
std::uint64_t n = 0;
for ([[maybe_unused]] Row<T> row : self.range) n++;
return n;
}
template<class Self> requires std::ranges::range<std::conditional_t<std::is_const_v<std::remove_reference_t<Self>>, const R, R>>
std::optional<Row<T>> First(this Self&& self) {
for (Row<T> row : self.range) return row;
return std::nullopt;
}
template<class Self> requires std::ranges::range<std::conditional_t<std::is_const_v<std::remove_reference_t<Self>>, const R, R>>
bool Any(this Self&& self) { return self.First().has_value(); }
template<class Pred>
auto Where(Pred pred) && {
return Query<T, std::ranges::filter_view<R, Pred>>(std::ranges::filter_view<R, Pred>(std::move(range), std::move(pred)));
}
template<class Pred>
auto Where(Pred pred) const& {
return Query<T, std::ranges::filter_view<R, Pred>>(std::ranges::filter_view<R, Pred>(range, std::move(pred)));
}
// Lazy view of one column's values (views for strings, values otherwise).
template<auto Ptr> requires std::is_same_v<MemberClass<Ptr>, T>
auto Select() && {
return std::views::transform(std::move(range), [](Row<T> row) { return row.template Get<Ptr>(); });
}
const R& Range() const noexcept { return range; }
private:
R range;
};
}

174
interfaces/Crafter.DB-Reflect.cppm Normal file
View file

@ -0,0 +1,174 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// Compile-time introspection of aggregates without P2996 (clang 23 has no
// <meta>). Three ingredients, all verified on the toolchain this library
// targets: structured-binding packs give member count and per-member access,
// std::source_location::function_name() leaks the member name when a pointer
// to a member of an extern object is used as a template argument, and address
// comparison on a constexpr instance maps a pointer-to-member to its index.
// When a clang ships <meta>, this partition is the only thing to replace.
export module Crafter.DB:Reflect;
import std;
namespace Crafter::DB::Reflect {
template<class T>
struct CountHelper {
template<class U>
static consteval auto Count(U& u) {
auto&& [...m] = u;
return std::integral_constant<std::size_t, sizeof...(m)>{};
}
static constexpr std::size_t value = decltype(Count(std::declval<T&>()))::value;
};
// A constant-initialised instance whose member addresses are turned into
// text by function_name(). Only its address is ever used, in a constant
// expression, so nothing reads or writes it at run time; no_destroy keeps
// the compiler from registering a destructor for it, which lets the linker
// drop the object entirely. (An undefined `extern` declaration would be
// cheaper still, but clang warns about it at every instantiation.)
template<class T>
[[clang::no_destroy]] inline const T external{};
template<auto P>
consteval std::string_view RawName() {
return std::source_location::current().function_name();
}
template<class T>
consteval std::string_view RawTypeName() {
return std::source_location::current().function_name();
}
}
export namespace Crafter::DB {
// True for class types whose non-static data members can be bound by a
// structured binding: public members, no bases carrying members, no
// user-declared constructors. Tables and nested column groups must be this.
template<class T>
concept Aggregate = std::is_aggregate_v<T> && !std::is_array_v<T> && requires(T& t) { Reflect::CountHelper<T>::value; };
template<Aggregate T>
inline constexpr std::size_t memberCount = Reflect::CountHelper<T>::value;
// Reference to the I-th member of an aggregate lvalue.
template<std::size_t I, class T>
constexpr decltype(auto) Member(T&& t) noexcept {
auto&& [...m] = t;
return (m...[I]);
}
template<std::size_t I, class T>
using MemberType = std::remove_cvref_t<decltype(Member<I>(std::declval<T&>()))>;
// The declared name of the I-th member, e.g. "iban". Parsed out of
// "std::string_view Crafter::DB::Reflect::RawName() [P = &external.iban]".
template<std::size_t I, Aggregate T>
consteval std::string_view MemberName() {
constexpr std::string_view raw = Reflect::RawName<&Member<I>(Reflect::external<T>)>();
const std::size_t end = raw.rfind(']');
const std::size_t dot = raw.rfind('.', end);
return raw.substr(dot + 1, end - dot - 1);
}
// The type's qualified name as the compiler spells it ("Nostro::Transaction").
// Used in diagnostics only, so the exact spelling is not load-bearing.
template<class T>
consteval std::string_view TypeName() {
constexpr std::string_view raw = Reflect::RawTypeName<T>();
const std::size_t end = raw.rfind(']');
const std::size_t eq = raw.rfind("T = ", end);
return raw.substr(eq + 4, end - eq - 4);
}
template<class Ptr>
struct MemberPointerTraits;
template<class C, class M>
struct MemberPointerTraits<M C::*> {
using Class = C;
using Type = M;
};
template<auto Ptr>
using MemberClass = typename MemberPointerTraits<decltype(Ptr)>::Class;
template<auto Ptr>
using MemberTypeOf = typename MemberPointerTraits<decltype(Ptr)>::Type;
inline constexpr std::size_t noMember = static_cast<std::size_t>(-1);
// Equality of two pointers-to-member that may point into different member
// types (comparing those directly is ill-formed): false unless the types
// match and the pointers are equal.
template<auto A, auto B>
consteval bool SameMemberPointer() {
if constexpr (std::is_same_v<std::remove_cv_t<decltype(A)>, std::remove_cv_t<decltype(B)>>) return A == B;
else return false;
}
// Index of the member a pointer-to-member designates, or noMember. Works
// by comparing the address of each bound member with the address the
// pointer selects on the same constexpr instance.
template<Aggregate T, auto Ptr>
consteval std::size_t MemberIndex() {
T t{};
std::size_t index = noMember;
[&]<std::size_t... I>(std::index_sequence<I...>) {
((void)[&] {
if constexpr (std::is_same_v<MemberType<I, T>, std::remove_cvref_t<decltype(t.*Ptr)>>) {
if (std::addressof(Member<I>(t)) == std::addressof(t.*Ptr)) index = I;
}
}(), ...);
}(std::make_index_sequence<memberCount<T>>{});
return index;
}
// Runtime counterpart of MemberIndex for a pointer that is not a template
// argument. Every call site passes a literal `&T::member`, and the probe
// object is constant-initialised (no guard to check), so after inlining the
// comparison chain folds to a constant.
template<Aggregate T, class M>
std::size_t MemberIndexOf(M T::* ptr) noexcept {
const T& instance = Reflect::external<T>;
std::size_t index = noMember;
[&]<std::size_t... I>(std::index_sequence<I...>) {
((void)[&] {
if constexpr (std::is_same_v<MemberType<I, T>, M>) {
if (std::addressof(Member<I>(instance)) == std::addressof(instance.*ptr)) index = I;
}
}(), ...);
}(std::make_index_sequence<memberCount<T>>{});
return index;
}
// Calls f.template operator()<I>(member) for every member, in order.
template<Aggregate T, class F>
constexpr void ForEachMember(T& t, F&& f) {
[&]<std::size_t... I>(std::index_sequence<I...>) {
(f.template operator()<I>(Member<I>(t)), ...);
}(std::make_index_sequence<memberCount<T>>{});
}
}
// Self-tests: a compiler whose function_name() text changes shape fails here,
// at the library's build, instead of producing garbage column names.
namespace Crafter::DB::Reflect {
struct Probe {
std::int32_t alpha;
double beta;
std::string gamma;
std::chrono::system_clock::time_point delta;
};
static_assert(Aggregate<Probe>);
static_assert(memberCount<Probe> == 4);
static_assert(MemberName<0, Probe>() == "alpha");
static_assert(MemberName<2, Probe>() == "gamma");
static_assert(MemberName<3, Probe>() == "delta");
static_assert(MemberIndex<Probe, &Probe::gamma>() == 2);
static_assert(MemberIndex<Probe, &Probe::delta>() == 3);
static_assert(std::is_same_v<MemberType<1, Probe>, double>);
static_assert(std::is_same_v<MemberTypeOf<&Probe::gamma>, std::string>);
static_assert(TypeName<Probe>().ends_with("Probe"));
}

277
interfaces/Crafter.DB-Row.cppm Normal file
View file

@ -0,0 +1,277 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// Row<T>: a 16-byte view of one stored row that decodes columns on demand,
// plus the ranges over rows: LiveRange skips tombstones, SlotView is random
// access over every slot (RowId space).
export module Crafter.DB:Row;
import std;
import :Reflect;
import :Format;
import :Formatter;
import :Schema;
import :Storage;
import :Codec;
export namespace Crafter::DB {
using RowId = std::uint64_t;
// Column tag: `inline constexpr Column<&Transaction::iban> iban{};`
// lets you write row[iban] and iban == x. The operators live in :Query.
template<auto Ptr>
struct Column {
using Table = MemberClass<Ptr>;
using Type = MemberTypeOf<Ptr>;
static constexpr auto pointer = Ptr;
static constexpr std::size_t index = MemberIndex<Table, Ptr>();
static constexpr std::string_view Name() noexcept { return MemberName<index, Table>(); }
};
template<TableType T>
class Row {
public:
Row() = default;
Row(const std::byte* row, const TableFile* file) noexcept : row(row), file(file) {}
// Positional access: member I, resolved at compile time.
template<std::size_t I>
View<MemberType<I, T>> Get() const {
return DecodeMember<T, I>(row, file->HeapView());
}
// Member-pointer access, resolved at compile time: row.Get<&T::iban>().
template<auto Ptr> requires std::is_same_v<MemberClass<Ptr>, T>
View<MemberTypeOf<Ptr>> Get() const {
constexpr std::size_t index = MemberIndex<T, Ptr>();
static_assert(index != noMember, "Crafter.DB: pointer does not designate a direct member of the table struct");
return Get<index>();
}
// Member-pointer access with a run-time pointer: row[&T::iban]. Every
// call site passes a literal, so the lookup folds after inlining.
template<class M>
View<M> operator[](M T::* ptr) const {
const std::size_t index = MemberIndexOf<T>(ptr);
View<M> result{};
[&]<std::size_t... I>(std::index_sequence<I...>) {
((void)[&] {
if constexpr (std::is_same_v<MemberType<I, T>, M>) {
if (index == I) result = Get<I>();
}
}(), ...);
}(std::make_index_sequence<memberCount<T>>{});
return result;
}
template<auto Ptr> requires std::is_same_v<MemberClass<Ptr>, T>
View<MemberTypeOf<Ptr>> operator[](Column<Ptr>) const {
return Get<Ptr>();
}
// Copies every column out into a T (allocates for strings and blobs).
T Materialize() const { return DecodeRow<T>(row, file->HeapView()); }
explicit operator T() const { return Materialize(); }
RowId Id() const noexcept { return static_cast<RowId>((row - file->Rows()) / Layout<T>::rowSize); }
bool Erased() const noexcept { return TableFile::Deleted(row); }
const std::byte* Bytes() const noexcept { return row; }
const TableFile* File() const noexcept { return file; }
friend bool operator==(const Row&, const Row&) = default;
private:
const std::byte* row = nullptr;
const TableFile* file = nullptr;
};
// Forward iterator over live rows of a slot range. The end of the range
// is a sentinel, so the iterator's own snapshot of the last slot is the
// only bound it ever compares against: a concurrent Append between two
// reads of the slot count cannot make it walk into unpublished memory.
// `checkFlags` is false when the table has no tombstones, which removes
// the per-row test.
template<TableType T>
class LiveIterator {
public:
using value_type = Row<T>;
using difference_type = std::ptrdiff_t;
using iterator_concept = std::forward_iterator_tag;
using iterator_category = std::forward_iterator_tag;
LiveIterator() = default;
LiveIterator(const std::byte* current, const std::byte* last, const TableFile* file, bool checkFlags) noexcept : current(current), last(last), file(file), checkFlags(checkFlags) { Skip(); }
Row<T> operator*() const noexcept { return { current, file }; }
LiveIterator& operator++() noexcept {
current += Layout<T>::rowSize;
Skip();
return *this;
}
LiveIterator operator++(int) noexcept {
LiveIterator copy = *this;
++*this;
return copy;
}
friend bool operator==(const LiveIterator& a, const LiveIterator& b) noexcept { return a.current == b.current; }
friend bool operator==(const LiveIterator& it, std::default_sentinel_t) noexcept { return it.current >= it.last; }
private:
void Skip() noexcept {
if (!checkFlags) return;
while (current < last && TableFile::Deleted(current)) current += Layout<T>::rowSize;
}
const std::byte* current = nullptr;
const std::byte* last = nullptr;
const TableFile* file = nullptr;
bool checkFlags = false;
};
template<TableType T>
class LiveRange : public std::ranges::view_interface<LiveRange<T>> {
public:
LiveRange() = default;
LiveRange(const TableFile* file, std::uint64_t firstSlot, std::uint64_t endSlot) noexcept
: file(file), first(file->Slot(firstSlot)), last(file->Slot(endSlot)) {}
// The tombstone flag is read when iteration starts, not when the range
// was built, so a query object created before an Erase still skips it.
LiveIterator<T> begin() const noexcept { return { first, last, file, file->HasTombstones() }; }
std::default_sentinel_t end() const noexcept { return {}; }
std::uint64_t SlotCount() const noexcept { return static_cast<std::uint64_t>((last - first) / Layout<T>::rowSize); }
private:
const TableFile* file = nullptr;
const std::byte* first = nullptr;
const std::byte* last = nullptr;
};
// Random access over every slot, tombstones included (row.Erased() tells).
template<TableType T>
class SlotView : public std::ranges::view_interface<SlotView<T>> {
public:
class Iterator {
public:
using value_type = Row<T>;
using difference_type = std::ptrdiff_t;
using iterator_concept = std::random_access_iterator_tag;
using iterator_category = std::random_access_iterator_tag;
Iterator() = default;
Iterator(const TableFile* file, std::uint64_t index) noexcept : file(file), index(index) {}
Row<T> operator*() const noexcept { return { file->Slot(index), file }; }
Row<T> operator[](difference_type n) const noexcept { return *(*this + n); }
Iterator& operator++() noexcept { ++index; return *this; }
Iterator operator++(int) noexcept { Iterator c = *this; ++index; return c; }
Iterator& operator--() noexcept { --index; return *this; }
Iterator operator--(int) noexcept { Iterator c = *this; --index; return c; }
Iterator& operator+=(difference_type n) noexcept { index = static_cast<std::uint64_t>(static_cast<difference_type>(index) + n); return *this; }
Iterator& operator-=(difference_type n) noexcept { return *this += -n; }
friend Iterator operator+(Iterator it, difference_type n) noexcept { return it += n; }
friend Iterator operator+(difference_type n, Iterator it) noexcept { return it += n; }
friend Iterator operator-(Iterator it, difference_type n) noexcept { return it -= n; }
friend difference_type operator-(const Iterator& a, const Iterator& b) noexcept { return static_cast<difference_type>(a.index) - static_cast<difference_type>(b.index); }
friend bool operator==(const Iterator& a, const Iterator& b) noexcept { return a.index == b.index; }
friend auto operator<=>(const Iterator& a, const Iterator& b) noexcept { return a.index <=> b.index; }
private:
const TableFile* file = nullptr;
std::uint64_t index = 0;
};
SlotView() = default;
SlotView(const TableFile* file, std::uint64_t firstSlot, std::uint64_t endSlot) noexcept : file(file), first(firstSlot), last(endSlot) {}
Iterator begin() const noexcept { return { file, first }; }
Iterator end() const noexcept { return { file, last }; }
std::size_t size() const noexcept { return static_cast<std::size_t>(last - first); }
Row<T> operator[](std::uint64_t index) const noexcept { return { file->Slot(first + index), file }; }
private:
const TableFile* file = nullptr;
std::uint64_t first = 0;
std::uint64_t last = 0;
};
// --- debug formatting ---------------------------------------------------
template<class V> struct IsBlobRef : std::false_type {};
template<class E> struct IsBlobRef<BlobRef<E>> : std::true_type {};
template<class V>
void FormatValue(std::string& out, const V& value) {
if constexpr (std::is_same_v<V, StringRef>) out += std::format("\"{}\"", value.View());
else if constexpr (std::is_same_v<V, std::string>) out += std::format("\"{}\"", value);
else if constexpr (std::is_same_v<V, bool>) out += value ? "true" : "false";
else if constexpr (std::is_enum_v<V>) out += std::format("{}", std::to_underlying(value));
else if constexpr (std::is_same_v<V, std::byte>) out += std::format("{}", std::to_integer<std::uint32_t>(value));
else if constexpr (std::is_arithmetic_v<V>) out += std::format("{}", value);
else if constexpr (IsBlobRef<V>::value) out += std::format("<{} elements>", value.Size());
else if constexpr (FixedStringTraits<V>::value) out += std::format("\"{}\"", value.View());
else if constexpr (OptionalTraits<V>::value) {
if (value) FormatValue(out, *value);
else out += "null";
}
else if constexpr (StdArrayTraits<V>::value) {
out += "[";
for (std::size_t i = 0; i < value.size(); i++) {
if (i != 0) out += ", ";
FormatValue(out, value[i]);
}
out += "]";
}
else if constexpr (std::formattable<V, char>) out += std::format("{}", value);
else if constexpr (Aggregate<V>) {
out += "{";
[&]<std::size_t... I>(std::index_sequence<I...>) {
((void)[&] {
if (I != 0) out += ", ";
out += MemberName<I, V>();
out += "=";
FormatValue(out, Member<I>(value));
}(), ...);
}(std::make_index_sequence<memberCount<V>>{});
out += "}";
}
else if constexpr (requires { value.size(); value.begin(); }) out += std::format("<{} elements>", value.size());
else out += "?";
}
template<TableType T>
std::string DescribeRow(Row<T> row) {
std::string out(TypeName<T>());
out += "{";
[&]<std::size_t... I>(std::index_sequence<I...>) {
((void)[&] {
if (I != 0) out += ", ";
out += MemberName<I, T>();
out += "=";
FormatValue(out, row.template Get<I>());
}(), ...);
}(std::make_index_sequence<memberCount<T>>{});
out += "}";
return out;
}
}
// Neither view owns the mapping, so iterators may outlive the view object.
template<class T>
inline constexpr bool std::ranges::enable_borrowed_range<Crafter::DB::SlotView<T>> = true;
template<class T>
inline constexpr bool std::ranges::enable_borrowed_range<Crafter::DB::LiveRange<T>> = true;
template<Crafter::DB::TableType T>
struct std::formatter<Crafter::DB::Row<T>> : std::formatter<std::string> {
auto format(const Crafter::DB::Row<T>& row, std::format_context& ctx) const {
return std::formatter<std::string>::format(Crafter::DB::DescribeRow(row), ctx);
}
};
template<>
struct std::formatter<Crafter::DB::StringRef> : std::formatter<std::string_view> {
auto format(const Crafter::DB::StringRef& ref, std::format_context& ctx) const {
return std::formatter<std::string_view>::format(ref.View(), ctx);
}
};

512
interfaces/Crafter.DB-Schema.cppm Normal file
View file

@ -0,0 +1,512 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// From a C++ struct to a row layout, entirely at compile time: members are
// classified by Shape, nested aggregates are flattened into dotted column
// names, columns are placed by ascending alignment after the flags byte at
// offset 0, and the result is hashed into the schema fingerprint the file
// header carries. The same partition holds the table-level declarations
// (version, orderedBy, Previous, Migrate) and the diagnostics for misuse.
export module Crafter.DB:Schema;
import std;
import :Reflect;
import :Format;
import :Formatter;
export namespace Crafter::DB {
enum class Shape : std::uint8_t { Scalar, FixedBytes, ScalarArray, Optional, Fixed, Var, Aggregate, Pointer, Unsupported };
template<class T> struct OptionalTraits : std::false_type {};
template<class M> struct OptionalTraits<std::optional<M>> : std::true_type { using Type = M; };
template<class T> struct StdArrayTraits : std::false_type {};
template<class E, std::size_t N> struct StdArrayTraits<std::array<E, N>> : std::true_type {
using Element = E;
static constexpr std::size_t size = N;
};
template<class T> struct FixedStringTraits : std::false_type {};
template<std::size_t N> struct FixedStringTraits<FixedString<N>> : std::true_type { static constexpr std::size_t size = N; };
template<class T>
consteval Shape ShapeOf() {
if constexpr (std::is_pointer_v<T> || std::is_member_pointer_v<T> || std::is_reference_v<T> || std::is_null_pointer_v<T>) return Shape::Pointer;
else if constexpr (Scalar<T>) return Shape::Scalar;
else if constexpr (FixedStringTraits<T>::value) return Shape::FixedBytes;
else if constexpr (StdArrayTraits<T>::value) {
if constexpr (ByteLike<typename StdArrayTraits<T>::Element>) return Shape::FixedBytes;
else if constexpr (Scalar<typename StdArrayTraits<T>::Element>) return Shape::ScalarArray;
else return Shape::Unsupported;
}
else if constexpr (OptionalTraits<T>::value) return Shape::Optional;
else if constexpr (VarFormatter<T>) return Shape::Var;
else if constexpr (FixedFormatter<T>) return Shape::Fixed;
else if constexpr (Aggregate<T>) return Shape::Aggregate;
else return Shape::Unsupported;
}
// Number of row columns a member of type T occupies.
template<class T>
consteval std::size_t LeafCount() {
constexpr Shape shape = ShapeOf<T>();
if constexpr (shape == Shape::Scalar || shape == Shape::FixedBytes || shape == Shape::Var) return 1;
else if constexpr (shape == Shape::ScalarArray) return StdArrayTraits<T>::size;
else if constexpr (shape == Shape::Optional) return 1 + LeafCount<typename OptionalTraits<T>::Type>();
else if constexpr (shape == Shape::Fixed) return LeafCount<typename Formatter<T>::Stored>();
else if constexpr (shape == Shape::Aggregate) {
return []<std::size_t... I>(std::index_sequence<I...>) { return (std::size_t(0) + ... + LeafCount<MemberType<I, T>>()); }(std::make_index_sequence<memberCount<T>>{});
}
else return 1; // unsupported shapes are rejected by CheckTable; count 1 keeps indices sane meanwhile
}
// First leaf of member I of aggregate T.
template<class T, std::size_t I>
consteval std::size_t LeafStart() {
return []<std::size_t... J>(std::index_sequence<J...>) { return (std::size_t(0) + ... + LeafCount<MemberType<J, T>>()); }(std::make_index_sequence<I>{});
}
// A compile-time string with value semantics (std::string cannot cross
// from consteval into a constexpr variable). Overflow is a build error,
// never a silent truncation: a truncated column name would alias another
// column, a truncated diagnostic would lose its hint.
template<std::size_t Capacity>
struct FixedText {
std::array<char, Capacity> chars{};
std::uint32_t length = 0;
constexpr void Append(std::string_view text) {
if (length + text.size() > chars.size()) throw "Crafter.DB: a column name or diagnostic exceeds the fixed buffer; shorten the member path";
for (const char c : text) chars[length++] = c;
}
constexpr std::string_view View() const noexcept { return { chars.data(), length }; }
constexpr std::size_t size() const noexcept { return length; }
constexpr const char* data() const noexcept { return chars.data(); }
};
using FixedName = FixedText<256>; // a dotted column path
using MessageText = FixedText<1024>; // a static_assert message
inline constexpr std::uint32_t noLeaf = 0xFFFFFFFF;
struct LeafInfo {
FixedName name;
Kind kind = Kind::UInt8;
std::uint8_t flags = 0;
std::uint16_t size = 1;
std::uint16_t align = 1;
std::uint32_t paramNum = 0;
std::uint32_t paramDen = 0;
std::uint32_t variant = 0;
std::uint32_t presenceLeaf = noLeaf;
};
template<class T>
consteval void CollectLeaves(std::vector<LeafInfo>& out, const FixedName& name, std::uint32_t presenceLeaf) {
constexpr Shape shape = ShapeOf<T>();
const std::uint8_t optionalFlag = presenceLeaf == noLeaf ? 0 : columnFlagOptional;
if constexpr (shape == Shape::Scalar) {
LeafInfo leaf{ name, ScalarKind<T>(), optionalFlag, sizeof(T), alignof(T) };
if constexpr (std::is_enum_v<T>) leaf.flags |= columnFlagEnum;
leaf.presenceLeaf = presenceLeaf;
out.push_back(leaf);
} else if constexpr (shape == Shape::FixedBytes) {
constexpr std::size_t n = [] {
if constexpr (FixedStringTraits<T>::value) return FixedStringTraits<T>::size;
else return StdArrayTraits<T>::size;
}();
static_assert(n > 0 && n <= 0xFFFF, "fixed byte columns must be 1..65535 bytes");
LeafInfo leaf{ name, Kind::FixedBytes, optionalFlag, static_cast<std::uint16_t>(n), 1, static_cast<std::uint32_t>(n), 0 };
leaf.presenceLeaf = presenceLeaf;
out.push_back(leaf);
} else if constexpr (shape == Shape::ScalarArray) {
for (std::size_t i = 0; i < StdArrayTraits<T>::size; i++) {
FixedName element = name;
element.Append("[");
char digits[20]{};
std::size_t len = 0;
std::size_t v = i;
do { digits[len++] = static_cast<char>('0' + v % 10); v /= 10; } while (v != 0);
for (std::size_t k = 0; k < len; k++) element.Append(std::string_view(&digits[len - 1 - k], 1));
element.Append("]");
CollectLeaves<typename StdArrayTraits<T>::Element>(out, element, presenceLeaf);
}
} else if constexpr (shape == Shape::Optional) {
FixedName presence = name;
presence.Append("?");
LeafInfo leaf{ presence, Kind::UInt8, static_cast<std::uint8_t>(columnFlagPresence | optionalFlag), 1, 1 };
leaf.presenceLeaf = presenceLeaf;
const std::uint32_t index = static_cast<std::uint32_t>(out.size());
out.push_back(leaf);
CollectLeaves<typename OptionalTraits<T>::Type>(out, name, index);
} else if constexpr (shape == Shape::Var) {
LeafInfo leaf{ name, Kind::String, optionalFlag, sizeof(Var), alignof(Var) };
if constexpr (HasFormatterKind<T>) leaf.kind = Formatter<T>::kind;
if constexpr (HasFormatterPeriod<T>) {
leaf.paramNum = Formatter<T>::paramNum;
leaf.paramDen = Formatter<T>::paramDen;
}
if constexpr (HasFormatterVariant<T>) leaf.variant = Formatter<T>::variant;
leaf.presenceLeaf = presenceLeaf;
out.push_back(leaf);
} else if constexpr (shape == Shape::Fixed) {
const std::size_t first = out.size();
using S = typename Formatter<T>::Stored;
CollectLeaves<S>(out, name, presenceLeaf);
if constexpr (HasFormatterKind<T>) {
static_assert(LeafCount<S>() == 1, "a Formatter may only override `kind` when Stored is a single column");
// A chrono type with a floating rep keeps its Float kind so the
// bytes are read back as what they are; the period still records
// that it is a time.
if constexpr (std::is_integral_v<S>) out[first].kind = Formatter<T>::kind;
}
if constexpr (HasFormatterPeriod<T>) {
out[first].paramNum = Formatter<T>::paramNum;
out[first].paramDen = Formatter<T>::paramDen;
}
if constexpr (HasFormatterVariant<T>) out[first].variant = Formatter<T>::variant;
} else if constexpr (shape == Shape::Aggregate) {
[&]<std::size_t... I>(std::index_sequence<I...>) {
((void)[&] {
FixedName member = name;
if (member.length != 0) member.Append(".");
member.Append(MemberName<I, T>());
CollectLeaves<MemberType<I, T>>(out, member, presenceLeaf);
}(), ...);
}(std::make_index_sequence<memberCount<T>>{});
} else {
LeafInfo leaf{ name, Kind::UInt8, optionalFlag, 1, 1 };
out.push_back(leaf);
}
}
template<class T>
consteval auto MakeLeaves() {
std::vector<LeafInfo> leaves;
CollectLeaves<T>(leaves, FixedName{}, noLeaf);
std::array<LeafInfo, LeafCount<T>()> out{};
for (std::size_t i = 0; i < out.size(); i++) out[i] = leaves[i];
return out;
}
// One placed column: a leaf with its row offset, or the flags byte.
struct ColumnInfo {
FixedName name;
Kind kind = Kind::Flags;
std::uint8_t flags = 0;
std::uint16_t size = 1;
std::uint16_t align = 1;
std::uint16_t offset = 0;
std::uint16_t presenceOffset = noColumn;
std::uint32_t paramNum = 0;
std::uint32_t paramDen = 0;
std::uint32_t variant = 0;
};
template<std::size_t LeafCountValue>
struct Arrangement {
std::array<ColumnInfo, LeafCountValue + 1> columns{};
std::array<std::uint16_t, LeafCountValue> leafOffset{};
std::array<std::uint16_t, LeafCountValue> leafColumn{};
std::uint32_t rowSize = 0;
std::uint16_t rowAlign = 1;
};
template<std::size_t N>
consteval Arrangement<N> Arrange(const std::array<LeafInfo, N>& leaves) {
Arrangement<N> result;
result.columns[0].name.Append("flags");
std::array<std::size_t, N> order{};
for (std::size_t i = 0; i < N; i++) order[i] = i;
std::ranges::stable_sort(order, [&](std::size_t a, std::size_t b) { return leaves[a].align < leaves[b].align; });
std::uint32_t cursor = 1;
std::uint16_t rowAlign = 1;
for (std::size_t k = 0; k < N; k++) {
const LeafInfo& leaf = leaves[order[k]];
const std::uint32_t align = leaf.align;
cursor = (cursor + align - 1) / align * align;
if (cursor + leaf.size > 0xFFFF) throw "a row must fit in 65535 bytes";
ColumnInfo& column = result.columns[k + 1];
column.name = leaf.name;
column.kind = leaf.kind;
column.flags = leaf.flags;
column.size = leaf.size;
column.align = leaf.align;
column.offset = static_cast<std::uint16_t>(cursor);
column.paramNum = leaf.paramNum;
column.paramDen = leaf.paramDen;
column.variant = leaf.variant;
result.leafOffset[order[k]] = static_cast<std::uint16_t>(cursor);
result.leafColumn[order[k]] = static_cast<std::uint16_t>(k + 1);
cursor += leaf.size;
if (leaf.align > rowAlign) rowAlign = leaf.align;
}
for (std::size_t k = 0; k < N; k++) {
const LeafInfo& leaf = leaves[order[k]];
if (leaf.presenceLeaf != noLeaf) result.columns[k + 1].presenceOffset = result.leafOffset[leaf.presenceLeaf];
}
result.rowAlign = rowAlign;
result.rowSize = (cursor + rowAlign - 1) / rowAlign * rowAlign;
return result;
}
template<std::size_t N>
consteval std::uint64_t FingerprintOf(const Arrangement<N>& arrangement, std::uint16_t orderedColumn) {
std::uint64_t hash = Fnv1a(fnvOffset, "crafter-db-v1");
hash = Fnv1a(hash, arrangement.rowSize);
hash = Fnv1a(hash, arrangement.rowAlign);
hash = Fnv1a(hash, orderedColumn);
for (const ColumnInfo& column : arrangement.columns) {
hash = Fnv1a(hash, column.name.View());
hash = Fnv1a(hash, static_cast<std::uint64_t>(column.kind));
hash = Fnv1a(hash, column.offset);
hash = Fnv1a(hash, column.size);
hash = Fnv1a(hash, column.flags);
hash = Fnv1a(hash, column.paramNum);
hash = Fnv1a(hash, column.paramDen);
hash = Fnv1a(hash, column.variant);
}
return hash;
}
template<std::size_t N>
consteval std::size_t NamePoolLength(const Arrangement<N>& arrangement) {
std::size_t total = 0;
for (const ColumnInfo& column : arrangement.columns) total += column.name.length;
return total;
}
// --- table-level declarations ---------------------------------------------
template<class T>
concept HasVersion = requires { { T::version } -> std::convertible_to<std::uint32_t>; } && std::is_integral_v<std::remove_cv_t<decltype(T::version)>>;
template<class T>
concept HasOrderedBy = requires { T::orderedBy; } && std::is_member_object_pointer_v<std::remove_cv_t<decltype(T::orderedBy)>>;
template<class T>
concept HasPrevious = requires { typename T::Previous; } && Aggregate<typename T::Previous>;
template<class T>
concept HasMigrate = HasPrevious<T> && requires(typename T::Previous&& previous) { { T::Migrate(std::move(previous)) } -> std::same_as<T>; };
// What `orderedBy` may point at: a type whose on-disk column is a single
// integer with a total order. Floats are out (NaN breaks the search),
// strings and fixed byte arrays are out (no binary search over them).
template<class M>
concept OrderableKey = (std::is_integral_v<M> && !std::is_same_v<M, bool>) || std::is_enum_v<M>
|| (FixedFormatter<M> && HasFormatterKind<M> && std::is_integral_v<typename Formatter<M>::Stored> && std::totally_ordered<M>);
// What a table struct must be. The diagnostics live in CheckTable.
template<class T>
concept TableType = Aggregate<T> && HasVersion<T> && std::is_default_constructible_v<T>;
// Everything the engine knows about T's row at compile time.
template<Aggregate T>
struct Layout {
static constexpr std::size_t leafCount = LeafCount<T>();
static constexpr std::size_t columnCount = leafCount + 1;
static constexpr std::array<LeafInfo, leafCount> leaves = MakeLeaves<T>();
static constexpr Arrangement<leafCount> arrangement = Arrange(leaves);
static constexpr const std::array<ColumnInfo, columnCount>& columns = arrangement.columns;
static constexpr std::uint32_t rowSize = arrangement.rowSize;
static constexpr std::uint16_t rowAlign = arrangement.rowAlign;
static_assert(columnCount <= 0xFFFF, "Crafter.DB: a table may have at most 65535 columns");
static_assert(NamePoolLength(arrangement) <= 0xFFFF, "Crafter.DB: the column names of a table may total at most 65535 bytes");
// Row offset of a leaf (leaf indices follow declaration order, depth first).
static constexpr std::uint16_t Offset(std::size_t leaf) noexcept { return arrangement.leafOffset[leaf]; }
static constexpr std::uint16_t Column(std::size_t leaf) noexcept { return arrangement.leafColumn[leaf]; }
static constexpr std::uint32_t version = [] {
if constexpr (HasVersion<T>) return static_cast<std::uint32_t>(T::version);
else return std::uint32_t(0);
}();
static constexpr std::size_t orderedMember = [] {
if constexpr (HasOrderedBy<T>) return MemberIndex<T, T::orderedBy>();
else return noMember;
}();
static constexpr std::uint16_t orderedColumn = [] {
if constexpr (HasOrderedBy<T>) return Column(LeafStart<T, MemberIndex<T, T::orderedBy>()>());
else return noColumn;
}();
static constexpr std::uint64_t fingerprint = FingerprintOf(arrangement, orderedColumn);
static constexpr bool hasHeap = std::ranges::any_of(columns, [](const ColumnInfo& c) { return c.kind == Kind::String || c.kind == Kind::Blob; });
};
// The file-side description of Layout<T>, byte-compatible with what
// SchemaInfo::Parse reads back; MakeSchemaInfo<T>().Fingerprint() equals
// Layout<T>::fingerprint by construction (a test pins this).
template<Aggregate T>
SchemaInfo MakeSchemaInfo() {
SchemaInfo info;
info.rowSize = Layout<T>::rowSize;
info.rowAlign = Layout<T>::rowAlign;
for (const ColumnInfo& column : Layout<T>::columns) {
ColumnDescriptor d{};
d.offset = column.offset;
d.size = column.size;
d.kind = column.kind;
d.flags = column.flags;
d.presenceOffset = column.presenceOffset;
d.nameOffset = static_cast<std::uint16_t>(info.namePool.size());
d.nameLength = static_cast<std::uint16_t>(column.name.length);
d.paramNum = column.paramNum;
d.paramDen = column.paramDen;
d.variant = column.variant;
info.namePool.append(column.name.View());
info.columns.push_back(d);
}
return info;
}
// --- diagnostics ---------------------------------------------------------
consteval MessageText Message(std::initializer_list<std::string_view> parts) {
MessageText message;
for (const std::string_view part : parts) message.Append(part);
return message;
}
template<class T, std::size_t I>
consteval MessageText MemberMessage(std::string_view problem, std::string_view hint) {
return Message({ "Crafter.DB: member '", MemberName<I, T>(), "' of ", TypeName<T>(), " ", problem, " ", hint });
}
template<class T>
consteval bool CheckMembers();
template<class T, std::size_t I>
consteval bool CheckMember() {
using M = MemberType<I, T>;
constexpr Shape shape = ShapeOf<M>();
static_assert(shape != Shape::Pointer, MemberMessage<T, I>("is a pointer or reference and cannot be stored;", "store a RowId, an index or a blob instead."));
static_assert(shape != Shape::Unsupported, MemberMessage<T, I>("has a type with no Crafter::DB::Formatter;", "specialise Crafter::DB::Formatter for it or make it an aggregate of storable members."));
if constexpr (shape == Shape::Aggregate) return CheckMembers<M>();
else if constexpr (shape == Shape::Optional) {
using P = typename OptionalTraits<M>::Type;
static_assert(ShapeOf<P>() != Shape::Pointer && ShapeOf<P>() != Shape::Unsupported, MemberMessage<T, I>("is a std::optional of an unsupported type;", "specialise Crafter::DB::Formatter for the payload type."));
static_assert(ShapeOf<P>() != Shape::Optional, MemberMessage<T, I>("is a std::optional of a std::optional;", "nested optionals have no distinct column names; wrap the inner one in a struct."));
if constexpr (ShapeOf<P>() == Shape::Aggregate) return CheckMembers<P>();
} else if constexpr (shape == Shape::Fixed) {
using S = typename Formatter<M>::Stored;
static_assert(ShapeOf<S>() != Shape::Pointer && ShapeOf<S>() != Shape::Unsupported && ShapeOf<S>() != Shape::Var && ShapeOf<S>() != Shape::Optional, MemberMessage<T, I>("has a Formatter whose Stored type is not storable;", "Stored must be a scalar, enum, std::array or aggregate of those."));
if constexpr (ShapeOf<S>() == Shape::Aggregate) return CheckMembers<S>();
}
return true;
}
template<class T>
consteval bool CheckMembers() {
return []<std::size_t... I>(std::index_sequence<I...>) { return (true && ... && CheckMember<T, I>()); }(std::make_index_sequence<memberCount<T>>{});
}
// Instantiated by Table<T>; every rule a table struct must follow is
// reported here with the table and member name in the message.
template<class T>
consteval bool CheckTable() {
static_assert(Aggregate<T>, Message({ "Crafter.DB: ", TypeName<T>(), " must be an aggregate: public data members only, no base classes with members, no user-declared constructors. Replace constructors with designated initialisers; Table::Emplace builds rows from fields." }));
static_assert(HasVersion<T>, Message({ "Crafter.DB: ", TypeName<T>(), " must declare `static constexpr std::uint32_t version = 1;`" }));
static_assert(std::is_default_constructible_v<T>, Message({ "Crafter.DB: ", TypeName<T>(), " must be default constructible (rows are decoded into a value-initialised instance)." }));
if constexpr (Aggregate<T>) {
CheckMembers<T>();
if constexpr (HasOrderedBy<T>) {
static_assert(std::is_same_v<MemberClass<T::orderedBy>, T>, Message({ "Crafter.DB: ", TypeName<T>(), "::orderedBy must point to a member of ", TypeName<T>() }));
static_assert(OrderableKey<MemberTypeOf<T::orderedBy>>, Message({ "Crafter.DB: ", TypeName<T>(), "::orderedBy must name an integer, enum or integral chrono member; floats (NaN has no order), strings, byte arrays and aggregates cannot be ordered." }));
}
if constexpr (HasPrevious<T>) {
static_assert(HasVersion<typename T::Previous>, Message({ "Crafter.DB: ", TypeName<T>(), "::Previous must itself declare `version`." }));
static_assert(Layout<typename T::Previous>::version < Layout<T>::version, Message({ "Crafter.DB: ", TypeName<T>(), "::Previous must have a lower version than ", TypeName<T>(), "; the Previous chain is strictly decreasing." }));
}
}
return true;
}
}
namespace Crafter::DB::SchemaTests {
enum class Currency : std::uint16_t { Euro };
struct Transaction {
std::string name;
std::string bic;
std::string iban;
std::string description;
Currency currency;
std::int64_t amount;
std::chrono::system_clock::time_point timePoint;
static constexpr std::uint32_t version = 1;
static constexpr auto orderedBy = &Transaction::timePoint;
};
static_assert(TableType<Transaction>);
static_assert(CheckTable<Transaction>());
static_assert(Layout<Transaction>::leafCount == 7);
static_assert(Layout<Transaction>::rowSize == 88);
static_assert(Layout<Transaction>::rowAlign == 8);
static_assert(Layout<Transaction>::columns[0].kind == Kind::Flags && Layout<Transaction>::columns[0].offset == 0);
static_assert(Layout<Transaction>::Offset(4) == 2); // currency: the only 2-byte column, right after the flags byte
static_assert(Layout<Transaction>::Offset(0) == 8); // name
static_assert(Layout<Transaction>::Offset(2) == 40); // iban
static_assert(Layout<Transaction>::Offset(5) == 72); // amount
static_assert(Layout<Transaction>::Offset(6) == 80); // timePoint
static_assert(Layout<Transaction>::columns[Layout<Transaction>::Column(6)].kind == Kind::TimePoint);
static_assert(Layout<Transaction>::columns[Layout<Transaction>::Column(6)].paramDen == 1000000);
static_assert(Layout<Transaction>::columns[Layout<Transaction>::Column(4)].flags == columnFlagEnum);
static_assert(Layout<Transaction>::columns[Layout<Transaction>::Column(2)].name.View() == "iban");
static_assert(Layout<Transaction>::orderedColumn == Layout<Transaction>::Column(6));
static_assert(Layout<Transaction>::hasHeap);
struct Money {
std::int64_t minor;
Currency currency;
};
struct Nested {
std::optional<std::string> note;
Money money;
std::array<std::uint16_t, 3> tags;
FixedString<11> bic;
std::vector<std::uint32_t> ids;
static constexpr std::uint32_t version = 2;
};
static_assert(CheckTable<Nested>());
static_assert(Layout<Nested>::leafCount == 2 + 2 + 3 + 1 + 1);
static_assert(Layout<Nested>::columns[Layout<Nested>::Column(0)].name.View() == "note?");
static_assert(Layout<Nested>::columns[Layout<Nested>::Column(1)].name.View() == "note");
static_assert(Layout<Nested>::columns[Layout<Nested>::Column(1)].flags == columnFlagOptional);
static_assert(Layout<Nested>::columns[Layout<Nested>::Column(1)].presenceOffset == Layout<Nested>::Offset(0));
static_assert(Layout<Nested>::columns[Layout<Nested>::Column(2)].name.View() == "money.minor");
static_assert(Layout<Nested>::columns[Layout<Nested>::Column(5)].name.View() == "tags[1]");
static_assert(Layout<Nested>::columns[Layout<Nested>::Column(7)].kind == Kind::FixedBytes && Layout<Nested>::columns[Layout<Nested>::Column(7)].size == 11);
static_assert(Layout<Nested>::columns[Layout<Nested>::Column(8)].kind == Kind::Blob && Layout<Nested>::columns[Layout<Nested>::Column(8)].paramDen == 4);
static_assert(Layout<Nested>::columns[Layout<Nested>::Column(8)].variant == static_cast<std::uint32_t>(Kind::UInt32));
struct FloatBlob {
std::vector<float> ids;
static constexpr std::uint32_t version = 1;
};
struct IntBlob {
std::vector<std::int32_t> ids;
static constexpr std::uint32_t version = 1;
};
static_assert(Layout<FloatBlob>::fingerprint != Layout<IntBlob>::fingerprint, "blob element type must be part of the fingerprint");
struct Unordered {
std::int64_t at;
std::string text;
static constexpr std::uint32_t version = 1;
};
struct Ordered {
std::int64_t at;
std::string text;
static constexpr std::uint32_t version = 1;
static constexpr auto orderedBy = &Ordered::at;
};
static_assert(Layout<Unordered>::fingerprint != Layout<Ordered>::fingerprint, "orderedBy must be part of the fingerprint");
}

201
interfaces/Crafter.DB-Storage.cppm Normal file
View file

@ -0,0 +1,201 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// The type-erased storage engine. A TableFile knows rows only as `rowSize`
// bytes and columns only through the descriptors the file carries, which is
// what lets it create, open, validate, recover, grow, compact and inspect any
// table without the C++ struct. Table<T> is a thin typed layer on top that
// computes column offsets at compile time and encodes/decodes through here.
export module Crafter.DB:Storage;
import std;
import :Error;
import :Format;
import :Formatter;
import :File;
export namespace Crafter::DB {
struct TableStats {
std::uint64_t slotCount = 0; // rows including tombstones
std::uint64_t liveCount = 0; // rows not deleted
std::uint64_t heapUsed = 0; // bytes of variable-length data written
std::uint64_t heapGarbage = 0; // of which orphaned by Erase/Update; Compact() reclaims
std::uint64_t fileLength = 0;
std::uint64_t commitSeq = 0;
};
struct CreateParams {
SchemaInfo schema;
std::uint32_t schemaVersion = 0;
std::uint64_t fingerprint = 0;
std::uint16_t orderedColumn = noColumn;
std::uint64_t initialRowBytes = 0; // size the first extents for a known payload (Compact, Migrate)
std::uint64_t initialHeapBytes = 0;
std::uint32_t migratedFromVersion = 0;
};
// Everything Inspect() can tell about a table without its struct.
struct Inspection {
FileHeader header{};
CommitRecord commit{};
SchemaInfo schema;
};
// One scalar column value read through a descriptor, for tools and Verify.
using ScalarValue = std::variant<std::int64_t, std::uint64_t, double>;
// A type-erased view of one row: column access by name or index through
// the file's own descriptors. This is what a dump tool or a migration from
// a struct that no longer exists uses.
class DynamicRow {
public:
DynamicRow() = default;
DynamicRow(const std::byte* row, const SchemaInfo* schema, const Heap* heap) noexcept : row(row), schema(schema), heap(heap) {}
bool Deleted() const noexcept;
std::size_t ColumnCount() const noexcept { return schema->columns.size(); }
const ColumnDescriptor& Column(std::size_t index) const noexcept { return schema->columns[index]; }
std::string_view Name(std::size_t index) const noexcept { return schema->Name(index); }
std::optional<std::size_t> Find(std::string_view name) const noexcept;
// False only for the payload of a disengaged std::optional.
bool Present(std::size_t index) const noexcept;
// Integers, bools, enums and chrono reps as int64/uint64; floats as double.
std::optional<ScalarValue> Scalar(std::size_t index) const noexcept;
// String, Blob and FixedBytes columns as raw bytes (bounds-checked).
std::optional<std::span<const std::byte>> Bytes(std::size_t index) const;
std::optional<std::string_view> Text(std::size_t index) const;
// "name=value, ..." for every column, for dumps and diagnostics.
std::string Describe() const;
private:
const std::byte* row = nullptr;
const SchemaInfo* schema = nullptr;
const Heap* heap = nullptr;
};
struct VerifyResult {
std::vector<std::string> problems;
bool Ok() const noexcept { return problems.empty(); }
};
class TableFile {
public:
// Writes a complete empty table to a temp file beside `path`, syncs it
// and renames it into place: a visible table file is always valid.
static void Create(const std::filesystem::path& path, const CreateParams& params, const OpenOptions& options);
static std::filesystem::path TempPath(const std::filesystem::path& path);
static void Replace(const std::filesystem::path& path, const std::filesystem::path& temp);
static std::filesystem::path BackupPath(const std::filesystem::path& path, std::uint32_t version);
static Inspection Inspect(const std::filesystem::path& path);
static void RemoveStaleTemps(const std::filesystem::path& path) noexcept;
TableFile(const std::filesystem::path& path, const OpenOptions& options);
TableFile(const TableFile&) = delete;
TableFile& operator=(const TableFile&) = delete;
~TableFile();
const FileHeader& Header() const noexcept { return header; }
const SchemaInfo& Schema() const noexcept { return schema; }
std::uint32_t RowSize() const noexcept { return header.rowSize; }
bool Writable() const noexcept { return writable; }
const std::filesystem::path& Path() const noexcept { return file.Path(); }
void SetPath(std::filesystem::path path) { file.SetPath(std::move(path)); }
TableStats Stats() const noexcept;
// Published counts. Readers take SlotCount() once per scan; rows below
// it are complete and never move.
std::uint64_t SlotCount() const noexcept { return publishedRows.load(std::memory_order_acquire); }
std::uint64_t LiveCount() const noexcept { return publishedLive.load(std::memory_order_relaxed); }
std::uint64_t HeapUsed() const noexcept { return publishedHeap.load(std::memory_order_acquire); }
// Set by the first Erase and never cleared while the file is open, so
// readers decide between the dense scan and the tombstone-checking
// scan from one monotonic flag rather than a racy pair of counters.
bool HasTombstones() const noexcept { return tombstones.load(std::memory_order_acquire); }
// False in a process that inherited this object through fork().
bool OwnedByThisProcess() const noexcept;
const std::byte* Rows() const noexcept { return rows.Base(); }
const std::byte* Slot(std::uint64_t index) const noexcept { return rows.Base() + index * header.rowSize; }
const Heap& HeapView() const noexcept { return heap; }
static bool Deleted(const std::byte* row) noexcept;
bool Deleted(std::uint64_t index) const noexcept { return Deleted(Slot(index)); }
// Append protocol: BeginAppend(n) returns the first of n writable slots
// (capacity guaranteed, contents undefined: callers encode into a
// zeroed buffer and copy). Variable-length bytes go through Writer().
// CommitAppend publishes; AbortAppend forgets the heap bytes.
std::byte* BeginAppend(std::uint64_t count);
HeapWriter& Writer() noexcept { return writer; }
void CommitAppend(std::uint64_t count) noexcept;
void AbortAppend() noexcept;
std::byte* MutableSlot(std::uint64_t index);
// Sets the tombstone bit. Returns false when the row was already deleted.
bool MarkDeleted(std::uint64_t index);
void AddHeapGarbage(std::uint64_t bytes) noexcept { heapGarbage += bytes; }
// Out-of-line bytes a row references (for garbage accounting).
std::uint64_t HeapBytesOf(const std::byte* row) const noexcept;
// Durability: fdatasync the data, then write the inactive commit slot.
void Flush(bool cleanClose = false);
void Close();
void RequireWritable() const;
// Streams the live rows into `target` (same schema, empty, writable),
// repacking their heap values in row order and dropping tombstones.
void CompactInto(TableFile& target) const;
DynamicRow Dynamic(std::uint64_t index) const noexcept { return { Slot(index), &schema, &heap }; }
// Full consistency scan: every handle inside the committed heap, the
// ordered column non-decreasing, live count equal to the recount,
// presence bytes 0 or 1. Returns the problems instead of throwing so a
// tool can print them all.
VerifyResult Verify() const;
private:
class Arena final : public HeapWriter {
public:
void Attach(TableFile* table, std::byte* arenaBase, std::uint64_t arenaCursor, std::uint64_t arenaCapacity) noexcept {
owner = table;
base = arenaBase;
cursor = arenaCursor;
capacity = arenaCapacity;
}
void SetCapacity(std::uint64_t arenaCapacity) noexcept { capacity = arenaCapacity; }
protected:
void Grow(std::uint64_t neededCapacity) override;
private:
TableFile* owner = nullptr;
};
void AddExtent(std::uint32_t region, std::uint64_t minimumBytes);
void EnsureHeapCapacity(std::uint64_t bytes);
void WriteCommit(bool cleanClose);
void RecountLive() noexcept;
File file;
OpenOptions options;
bool writable = false;
bool closed = false;
FileHeader header{};
SchemaInfo schema;
CommitRecord current{};
std::size_t activeSlot = 0;
MappedPages headerPages;
MappedRegion rows;
MappedRegion heapRegion;
Heap heap;
Arena writer;
std::atomic<std::uint64_t> publishedRows{ 0 };
std::atomic<std::uint64_t> publishedLive{ 0 };
std::atomic<std::uint64_t> publishedHeap{ 0 };
std::atomic<bool> tombstones{ false };
std::uint64_t heapGarbage = 0;
std::uint64_t fileLength = 0;
std::int64_t ownerProcess = 0;
std::array<std::uint32_t, regionSlots> extentCount{};
std::vector<std::size_t> varColumns;
};
}

440
interfaces/Crafter.DB-Table.cppm Normal file
View file

@ -0,0 +1,440 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// Table<T>: the user-facing handle. Opening creates, validates and migrates;
// writes go through the typed encoders into the mapped file; reads are
// zero-copy Row<T> views; queries are lambdas or column expressions.
export module Crafter.DB:Table;
import std;
import :Error;
import :Reflect;
import :Format;
import :Formatter;
import :Schema;
import :File;
import :Storage;
import :Codec;
import :Row;
import :Query;
import :Migration;
export namespace Crafter::DB {
template<class T>
class Table {
static_assert(CheckTable<T>());
public:
using Value = T;
explicit Table(const std::filesystem::path& path, OpenOptions options = {}) : path(path), options(options) {
Open();
}
Table(Table&&) noexcept = default;
Table& operator=(Table&&) noexcept = default;
Table(const Table&) = delete;
Table& operator=(const Table&) = delete;
// --- information ------------------------------------------------------
const std::filesystem::path& Path() const noexcept { return path; }
TableStats Stats() const noexcept { return file->Stats(); }
// Live rows.
std::uint64_t Size() const noexcept { return file->LiveCount(); }
// Slots, tombstones included: the range of valid RowIds.
std::uint64_t SlotCount() const noexcept { return file->SlotCount(); }
bool Empty() const noexcept { return Size() == 0; }
bool ReadOnly() const noexcept { return !file->Writable(); }
TableFile& File() noexcept { return *file; }
const TableFile& File() const noexcept { return *file; }
// --- writing ----------------------------------------------------------
RowId Append(const T& row) {
const RowId id = AppendRows<T>(*file, std::span<const T>(&row, 1));
AutoFlush();
return id;
}
template<std::ranges::input_range R> requires std::convertible_to<std::ranges::range_reference_t<R>, const T&>
RowId Append(R&& rows) {
const RowId id = AppendRows<T>(*file, std::forward<R>(rows));
AutoFlush();
return id;
}
// One argument per member, in declaration order, encoded straight into
// the columns (a std::string_view is enough for a std::string member).
template<class... Args>
RowId Emplace(const Args&... args) {
static_assert(sizeof...(Args) == memberCount<T>, "Crafter.DB: Emplace takes exactly one argument per member of the table struct, in declaration order");
file->RequireWritable();
const RowId id = file->SlotCount();
if constexpr (HasOrderedBy<T>) {
std::optional<std::uint64_t> before;
if (id != 0) before = id - 1;
const OrderedKey<T> key = static_cast<OrderedKey<T>>(std::get<Layout<T>::orderedMember>(std::tie(args...)));
CheckOrder<T>(*file, key, before, std::nullopt);
}
std::byte* out = file->BeginAppend(1);
try {
EmplaceRow<T>(out, file->Writer(), args...);
} catch (...) {
file->AbortAppend();
throw;
}
file->CommitAppend(1);
AutoFlush();
return id;
}
// Replaces a live row in place. The new heap bytes are committed to
// disk before the row starts pointing at them, so a crash leaves
// either the old row or the new one, never a dangling handle. The row
// bytes themselves are not atomic against a power loss mid-write.
void Update(RowId id, const T& row) {
file->RequireWritable();
RequireLive(id);
if constexpr (HasOrderedBy<T>) CheckNeighbours(id, row.*T::orderedBy);
alignas(8) std::array<std::byte, Layout<T>::rowSize> buffer;
HeapWriter& heap = file->Writer();
const std::uint64_t heapBefore = heap.Cursor();
try {
EncodeRow<T>(row, buffer.data(), heap);
} catch (...) {
file->AbortAppend();
throw;
}
if (heap.Cursor() != heapBefore) {
file->CommitAppend(0);
file->Flush(false);
}
std::byte* slot = file->MutableSlot(id);
file->AddHeapGarbage(file->HeapBytesOf(slot));
std::memcpy(slot, buffer.data(), Layout<T>::rowSize);
AutoFlush();
}
// Sets one member of a live row, with the same durability order as
// Update. Only that member's columns are written, so a concurrent
// Erase of the row is not undone.
template<auto Ptr, class V> requires std::is_same_v<MemberClass<Ptr>, T>
void Set(RowId id, const V& value) {
constexpr std::size_t index = MemberIndex<T, Ptr>();
static_assert(index != noMember, "Crafter.DB: Set needs a pointer to a direct member of the table struct");
constexpr std::size_t first = LeafStart<T, index>();
constexpr std::size_t count = LeafCount<MemberType<index, T>>();
file->RequireWritable();
RequireLive(id);
if constexpr (HasOrderedBy<T>) {
if constexpr (SameMemberPointer<Ptr, T::orderedBy>()) CheckNeighbours(id, static_cast<OrderedKey<T>>(value));
}
std::byte* slot = file->MutableSlot(id);
alignas(8) std::array<std::byte, Layout<T>::rowSize> buffer;
std::memcpy(buffer.data(), slot, Layout<T>::rowSize);
HeapWriter& heap = file->Writer();
const std::uint64_t heapBefore = heap.Cursor();
try {
EncodeMember<T, index>(buffer.data(), heap, value);
} catch (...) {
file->AbortAppend();
throw;
}
if (heap.Cursor() != heapBefore) {
file->CommitAppend(0);
file->Flush(false);
}
file->AddHeapGarbage(MemberHeapBytes<T, index>(slot));
CopyLeaves<T, first, count>(slot, buffer.data());
AutoFlush();
}
template<auto Ptr, class V> requires std::is_same_v<MemberClass<Ptr>, T>
void Set(RowId id, Column<Ptr>, const V& value) {
Set<Ptr>(id, value);
}
// Tombstones the row. Returns false when it was already erased.
bool Erase(RowId id) {
file->RequireWritable();
RequireSlot(id);
const bool erased = file->MarkDeleted(id);
if (erased) AutoFlush();
return erased;
}
template<class Pred>
std::uint64_t EraseIf(Pred pred) {
file->RequireWritable();
std::uint64_t erased = 0;
const std::uint64_t slots = file->SlotCount();
for (std::uint64_t i = 0; i < slots; i++) {
const Row<T> row(file->Slot(i), file.get());
if (row.Erased() || !pred(row)) continue;
if (file->MarkDeleted(i)) erased++;
}
if (erased != 0) AutoFlush();
return erased;
}
void Flush() { file->Flush(false); }
// Flushes and closes now, reporting failure as an exception. The
// destructor does the same but cannot report, so call this when a
// failed final flush must not go unnoticed. The table is unusable after.
void Close() { file->Close(); }
// Consistency scan of the whole file (see TableFile::Verify).
VerifyResult Verify() const { return file->Verify(); }
// Rewrites the file without tombstones and heap garbage. Invalidates
// every RowId, Row and view taken before the call.
void Compact() {
file->RequireWritable();
const TableStats stats = file->Stats();
Rebuild([&](TableFile& target) { file->CompactInto(target); }, stats.liveCount * Layout<T>::rowSize, stats.heapUsed - std::min(stats.heapUsed, stats.heapGarbage));
}
// Removes every row. Invalidates RowIds, Rows and views.
void Clear() {
file->RequireWritable();
Rebuild([](TableFile&) {}, 0, 0);
}
// --- reading ----------------------------------------------------------
Row<T> operator[](RowId id) const noexcept { return { file->Slot(id), file.get() }; }
Row<T> At(RowId id) const {
RequireLive(id);
return (*this)[id];
}
bool Contains(RowId id) const noexcept { return id < file->SlotCount() && !file->Deleted(id); }
Row<T> Front() const {
for (Row<T> row : Live()) return row;
throw std::out_of_range("Table::Front on an empty table");
}
Row<T> Back() const {
std::uint64_t index = file->SlotCount();
while (index != 0) {
index--;
if (!file->Deleted(index)) return (*this)[index];
}
throw std::out_of_range("Table::Back on an empty table");
}
LiveRange<T> Live() const noexcept { return { file.get(), 0, file->SlotCount() }; }
LiveIterator<T> begin() const noexcept { return Live().begin(); }
std::default_sentinel_t end() const noexcept { return {}; }
SlotView<T> Slots() const noexcept { return { file.get(), 0, file->SlotCount() }; }
std::vector<T> Materialize() const {
std::vector<T> out;
out.reserve(Size());
for (Row<T> row : Live()) out.push_back(row.Materialize());
return out;
}
// Live rows of the slot range [first, last).
Query<T, LiveRange<T>> Rows(RowId first, RowId last) const noexcept {
const std::uint64_t end = std::min<std::uint64_t>(last, file->SlotCount());
return Query<T, LiveRange<T>>(LiveRange<T>(file.get(), std::min(first, end), end));
}
// Lazy filtered view. For a column expression on an orderedBy table the
// bounds on the ordered column are turned into a binary search first.
template<class Pred>
auto Where(Pred pred) const {
std::uint64_t first = 0;
std::uint64_t last = file->SlotCount();
if constexpr (HasOrderedBy<T> && Expression<Pred>) {
Bounds<OrderedKey<T>> bounds;
CollectBounds<T>(pred, bounds);
if (bounds.lower) first = bounds.lowerExclusive ? UpperBoundSlot<T>(*file, *bounds.lower, 0, last) : LowerBoundSlot<T>(*file, *bounds.lower, 0, last);
if (bounds.upper) last = bounds.upperExclusive ? LowerBoundSlot<T>(*file, *bounds.upper, first, last) : UpperBoundSlot<T>(*file, *bounds.upper, first, last);
if (last < first) last = first;
}
return Query<T, std::ranges::filter_view<LiveRange<T>, Pred>>(std::ranges::filter_view<LiveRange<T>, Pred>(LiveRange<T>(file.get(), first, last), std::move(pred)));
}
template<auto Ptr> requires std::is_same_v<MemberClass<Ptr>, T>
auto Select() const {
return std::views::transform(Live(), [](Row<T> row) { return row.template Get<Ptr>(); });
}
// --- ordered column ---------------------------------------------------
template<auto Ptr>
RowId LowerBound(const MemberTypeOf<Ptr>& key) const {
RequireOrdered<Ptr>();
return LowerBoundSlot<T>(*file, key, 0, file->SlotCount());
}
template<auto Ptr>
RowId UpperBound(const MemberTypeOf<Ptr>& key) const {
RequireOrdered<Ptr>();
return UpperBoundSlot<T>(*file, key, 0, file->SlotCount());
}
// Live rows with from <= key <= to.
template<auto Ptr>
Query<T, LiveRange<T>> Range(const MemberTypeOf<Ptr>& from, const MemberTypeOf<Ptr>& to) const {
RequireOrdered<Ptr>();
const std::uint64_t slots = file->SlotCount();
const std::uint64_t first = LowerBoundSlot<T>(*file, from, 0, slots);
const std::uint64_t last = UpperBoundSlot<T>(*file, to, first, slots);
return Rows(first, last);
}
// key >= from
template<auto Ptr>
Query<T, LiveRange<T>> From(const MemberTypeOf<Ptr>& from) const {
RequireOrdered<Ptr>();
return Rows(LowerBoundSlot<T>(*file, from, 0, file->SlotCount()), file->SlotCount());
}
// key > after
template<auto Ptr>
Query<T, LiveRange<T>> After(const MemberTypeOf<Ptr>& after) const {
RequireOrdered<Ptr>();
return Rows(UpperBoundSlot<T>(*file, after, 0, file->SlotCount()), file->SlotCount());
}
// key <= to
template<auto Ptr>
Query<T, LiveRange<T>> Until(const MemberTypeOf<Ptr>& to) const {
RequireOrdered<Ptr>();
return Rows(0, UpperBoundSlot<T>(*file, to, 0, file->SlotCount()));
}
// key < before
template<auto Ptr>
Query<T, LiveRange<T>> Before(const MemberTypeOf<Ptr>& before) const {
RequireOrdered<Ptr>();
return Rows(0, LowerBoundSlot<T>(*file, before, 0, file->SlotCount()));
}
private:
template<auto Ptr>
static consteval void RequireOrdered() {
static_assert(HasOrderedBy<T>, "Crafter.DB: Range/From/After/Until/Before/LowerBound/UpperBound need `static constexpr auto orderedBy = &T::member;` on the table struct");
if constexpr (HasOrderedBy<T>) {
static_assert(SameMemberPointer<Ptr, T::orderedBy>(), "Crafter.DB: binary search is only available on the member named by `orderedBy`; use Where() for other columns");
}
}
void CreateFile() {
CreateParams params;
params.schema = MakeSchemaInfo<T>();
params.schemaVersion = Layout<T>::version;
params.fingerprint = Layout<T>::fingerprint;
params.orderedColumn = Layout<T>::orderedColumn;
TableFile::Create(path, params, options); // IOError(file_exists) when the file is already there
}
// One open() does the work: the file's own header says whether it is
// current (done) or older (migrate, then reopen). A missing file is
// created with a rename that never replaces, so two processes racing
// to create the same table end up sharing one file.
void Open() {
if (options.mode == OpenMode::CreateNew) {
CreateFile();
file = std::make_unique<TableFile>(path, options);
return;
}
std::unique_ptr<TableFile> opened;
for (std::int32_t attempt = 0; !opened; attempt++) {
try {
opened = std::make_unique<TableFile>(path, options);
} catch (const IOError& error) {
if (error.code != std::errc::no_such_file_or_directory || options.mode == OpenMode::OpenExisting || options.readOnly || attempt != 0) throw;
try {
CreateFile();
file = std::make_unique<TableFile>(path, options);
return;
} catch (const IOError& created) {
if (created.code != std::errc::file_exists) throw;
// Lost the race: another process created it; open theirs.
}
}
}
if (opened->Header().schemaVersion != Layout<T>::version) {
Inspection info;
info.header = opened->Header();
info.schema = opened->Schema();
opened.reset(); // the migration takes the lock itself
MigrateTable<T>(path, info, options);
opened = std::make_unique<TableFile>(path, options);
if (opened->Header().schemaVersion != Layout<T>::version) {
throw SchemaError(std::format("{}: version {} on disk after migration, expected {}", path.string(), opened->Header().schemaVersion, Layout<T>::version));
}
}
RequireFingerprint<T>(path, opened->Header(), opened->Schema());
file = std::move(opened);
}
void AutoFlush() {
if (options.durability == Durability::EveryAppend) file->Flush(false);
}
void RequireSlot(RowId id) const {
if (id >= file->SlotCount()) throw std::out_of_range(std::format("RowId {} is outside the table ({} slots)", id, file->SlotCount()));
}
void RequireLive(RowId id) const {
RequireSlot(id);
if (file->Deleted(id)) throw std::out_of_range(std::format("RowId {} was erased", id));
}
template<class Key>
void CheckNeighbours(RowId id, const Key& key) {
std::optional<std::uint64_t> before;
std::optional<std::uint64_t> after;
if (id != 0) before = id - 1;
if (id + 1 < file->SlotCount()) after = id + 1;
CheckOrder<T>(*file, key, before, after);
}
template<class Fill>
void Rebuild(Fill&& fill, std::uint64_t initialRowBytes, std::uint64_t initialHeapBytes) {
CreateParams params;
params.schema = file->Schema();
params.schemaVersion = Layout<T>::version;
params.fingerprint = Layout<T>::fingerprint;
params.orderedColumn = Layout<T>::orderedColumn;
params.initialRowBytes = initialRowBytes;
params.initialHeapBytes = initialHeapBytes;
params.migratedFromVersion = file->Header().migratedFromVersion;
OpenOptions targetOptions = options;
targetOptions.createDirectories = false;
const std::filesystem::path temp = TableFile::TempPath(path);
TableFile::Create(temp, params, targetOptions);
std::unique_ptr<TableFile> target;
try {
target = std::make_unique<TableFile>(temp, targetOptions);
fill(*target);
target->Flush(false);
TableFile::Replace(path, temp);
} catch (...) {
target.reset();
Os::Remove(temp);
throw;
}
target->SetPath(path);
// The old inode is unlinked; a failing final flush of it is
// irrelevant and must not leave this Table without a file.
try {
file->Close();
} catch (...) {
}
file = std::move(target);
}
std::unique_ptr<TableFile> file;
std::filesystem::path path;
OpenOptions options;
};
}

17
interfaces/Crafter.DB.cppm Normal file
View file

@ -0,0 +1,17 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
export module Crafter.DB;
export import :Error;
export import :Reflect;
export import :Format;
export import :Formatter;
export import :Schema;
export import :File;
export import :Storage;
export import :Codec;
export import :Row;
export import :Query;
export import :Migration;
export import :Table;