Catcrafts/Crafter.DB
Watch
2
0
Fork
You've already forked Crafter.DB
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Initial commit: Crafter.DB

Browse source 
Header-only C++26 module library for typed, file-backed tables with
schema reflection, migrations, queries, and crash-safe commits.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Jorijn van der Graaf 2026-10-17 04:16:51 +02:00
commit bfd0cc261b
Signed by: jorijnvdgraaf
GPG key ID: 2937E59CDCC1BCFB
34 changed files with 7506 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

212
interfaces/Crafter.DB-Migration.cppm Normal file
View file

@ -0,0 +1,212 @@
// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
// Schema migrations in C++. A table struct names its predecessor with
// `using Previous = OldStruct;` and either supplies
// `static T Migrate(OldStruct&&)` or, for purely additive changes, nothing:
// members are then copied by name and new ones value-initialised. Opening a
// file whose version is older than the struct's walks the chain to the type
// with that version and streams every live row through the conversions into
// a new file that replaces the old one atomically.
export module Crafter.DB:Migration;
import std;
import :Error;
import :Reflect;
import :Format;
import :Formatter;
import :Schema;
import :File;
import :Storage;
import :Codec;
export namespace Crafter::DB {
template<class From, std::size_t I, class To>
consteval std::size_t FindMemberByName() {
constexpr std::string_view name = MemberName<I, From>();
std::size_t found = noMember;
[&]<std::size_t... J>(std::index_sequence<J...>) {
((void)[&] { if (MemberName<J, To>() == name) found = J; }(), ...);
}(std::make_index_sequence<memberCount<To>>{});
return found;
}
// Additive migration: every member of From must exist in To with the same
// name and type; members only in To are value-initialised.
template<Aggregate To, Aggregate From>
To CopyByName(From&& from) {
To to{};
[&]<std::size_t... I>(std::index_sequence<I...>) {
((void)[&] {
constexpr std::size_t j = FindMemberByName<From, I, To>();
static_assert(j != noMember, Message({ "Crafter.DB: member '", MemberName<I, From>(), "' of ", TypeName<From>(), " has no member of the same name in ", TypeName<To>(), "; a removed or renamed column needs an explicit `static ", TypeName<To>(), " Migrate(", TypeName<From>(), "&&)`." }));
if constexpr (j != noMember) {
static_assert(std::is_same_v<MemberType<I, From>, MemberType<j, To>>, Message({ "Crafter.DB: member '", MemberName<I, From>(), "' changed type between ", TypeName<From>(), " and ", TypeName<To>(), "; a type change needs an explicit `static ", TypeName<To>(), " Migrate(", TypeName<From>(), "&&)`." }));
Member<j>(to) = std::move(Member<I>(from));
}
}(), ...);
}(std::make_index_sequence<memberCount<From>>{});
return to;
}
// One step up the chain: Previous -> Next.
template<class Next>
Next ConvertStep(typename Next::Previous&& from) {
if constexpr (HasMigrate<Next>) return Next::Migrate(std::move(from));
else return CopyByName<Next>(std::move(from));
}
// The type directly above `From` in To's chain.
template<class To, class From>
consteval auto NextAbove() {
if constexpr (std::is_same_v<typename To::Previous, From>) return std::type_identity<To>{};
else return NextAbove<typename To::Previous, From>();
}
template<class To, class From>
To ConvertChain(From&& from) {
if constexpr (std::is_same_v<From, To>) return std::move(from);
else {
using Next = typename decltype(NextAbove<To, From>())::type;
Next next = ConvertStep<Next>(std::move(from));
return ConvertChain<To, Next>(std::move(next));
}
}
std::string DescribeColumns(const SchemaInfo& schema) {
std::string out = "[";
for (std::size_t i = 1; i < schema.columns.size(); i++) {
if (i != 1) out += ", ";
out += std::format("{}:{}", schema.Name(i), KindName(schema.columns[i].kind));
}
return out + "]";
}
template<Aggregate T>
void RequireFingerprint(const std::filesystem::path& path, const FileHeader& header, const SchemaInfo& schema) {
if (header.schemaFingerprint == Layout<T>::fingerprint && header.rowSize == Layout<T>::rowSize && header.orderedColumn == Layout<T>::orderedColumn) return;
const std::string onDisk = DescribeColumns(schema);
const std::string compiled = DescribeColumns(MakeSchemaInfo<T>());
if (onDisk == compiled && header.orderedColumn != Layout<T>::orderedColumn) {
const auto name = [&](std::uint16_t column) { return column == noColumn ? std::string("none") : std::string(schema.Name(column)); };
throw SchemaError(std::format("{}: {} version {} on disk is ordered by {} but the compiled struct is ordered by {}: `orderedBy` changed without bumping `version`", path.string(), TypeName<T>(), header.schemaVersion, name(header.orderedColumn), name(Layout<T>::orderedColumn)));
}
throw SchemaError(std::format("{}: {} version {} on disk has columns {} but the compiled struct has {}: the struct changed without bumping `version`", path.string(), TypeName<T>(), header.schemaVersion, onDisk, compiled));
}
// Writes the converted live rows of `old` into a fresh file and returns
// its path. Streaming keeps memory flat; `sorted` collects everything and
// stable-sorts by the new orderedBy first, for the case where the old data
// is not in the order the new struct declares.
template<TableType T, Aggregate Old>
std::filesystem::path WriteConverted(const TableFile& old, const std::filesystem::path& path, const OpenOptions& options, bool sorted) {
const TableStats stats = old.Stats();
CreateParams params;
params.schema = MakeSchemaInfo<T>();
params.schemaVersion = Layout<T>::version;
params.fingerprint = Layout<T>::fingerprint;
params.orderedColumn = Layout<T>::orderedColumn;
params.initialRowBytes = stats.liveCount * Layout<T>::rowSize;
params.initialHeapBytes = stats.heapUsed - std::min(stats.heapUsed, stats.heapGarbage);
params.migratedFromVersion = old.Header().schemaVersion;
OpenOptions targetOptions = options;
targetOptions.readOnly = false;
targetOptions.createDirectories = false;
const std::filesystem::path temp = TableFile::TempPath(path);
TableFile::Create(temp, params, targetOptions);
try {
TableFile target(temp, targetOptions);
const std::uint64_t slots = old.SlotCount();
if (sorted) {
std::vector<T> rows;
rows.reserve(stats.liveCount);
for (std::uint64_t i = 0; i < slots; i++) {
const std::byte* row = old.Slot(i);
if (!TableFile::Deleted(row)) rows.push_back(ConvertChain<T, Old>(DecodeRow<Old>(row, old.HeapView())));
}
if constexpr (HasOrderedBy<T>) std::ranges::stable_sort(rows, {}, [](const T& row) { return row.*T::orderedBy; });
AppendRows<T>(target, std::span<const T>(rows));
} else {
for (std::uint64_t i = 0; i < slots; i++) {
const std::byte* row = old.Slot(i);
if (TableFile::Deleted(row)) continue;
T converted = ConvertChain<T, Old>(DecodeRow<Old>(row, old.HeapView()));
AppendRows<T>(target, std::span<const T>(&converted, 1));
}
}
target.Flush(true);
} catch (...) {
Os::Remove(temp);
throw;
}
return temp;
}
template<TableType T, Aggregate Old>
void RewriteFrom(const std::filesystem::path& path, const OpenOptions& options) {
OpenOptions sourceOptions = options;
sourceOptions.readOnly = false;
sourceOptions.mode = OpenMode::OpenExisting;
TableFile old(path, sourceOptions);
const std::uint32_t oldVersion = old.Header().schemaVersion;
std::filesystem::path temp;
try {
temp = WriteConverted<T, Old>(old, path, options, false);
} catch (const OrderError&) {
if constexpr (!HasOrderedBy<T>) throw;
temp = WriteConverted<T, Old>(old, path, options, true);
}
try {
if (options.backupBeforeMigrate) {
const std::filesystem::path backup = TableFile::BackupPath(path, oldVersion);
Os::Remove(backup);
Os::HardLink(path, backup);
}
old.Close();
TableFile::Replace(path, temp);
} catch (...) {
Os::Remove(temp);
throw;
}
}
template<TableType T, Aggregate Old>
bool WalkChain(const std::filesystem::path& path, const Inspection& info, const OpenOptions& options) {
if (Layout<Old>::version == info.header.schemaVersion) {
RequireFingerprint<Old>(path, info.header, info.schema);
RewriteFrom<T, Old>(path, options);
return true;
}
if constexpr (HasPrevious<Old>) return WalkChain<T, typename Old::Previous>(path, info, options);
else return false;
}
template<class T>
consteval std::uint32_t OldestVersion() {
if constexpr (HasPrevious<T>) return OldestVersion<typename T::Previous>();
else return Layout<T>::version;
}
// Brings a file whose version differs from T's up to date, or throws.
template<TableType T>
void MigrateTable(const std::filesystem::path& path, const Inspection& info, const OpenOptions& options) {
const std::uint32_t fileVersion = info.header.schemaVersion;
if (fileVersion == Layout<T>::version) return;
if (fileVersion > Layout<T>::version) throw SchemaError(std::format("{}: file is at version {} but this build of {} is version {}; the file was written by a newer build", path.string(), fileVersion, TypeName<T>(), Layout<T>::version));
if (options.readOnly) throw MigrationError(std::format("{}: file is at version {} and {} is version {}, but the table was opened read-only", path.string(), fileVersion, TypeName<T>(), Layout<T>::version));
if (!options.migrate) throw MigrationError(std::format("{}: file is at version {} and {} is version {}, and migration was disabled", path.string(), fileVersion, TypeName<T>(), Layout<T>::version));
if (!WalkChain<T, T>(path, info, options)) {
throw SchemaError(std::format("{}: no migration path from on-disk version {} to {}: the Previous chain of {} stops at version {}", path.string(), fileVersion, Layout<T>::version, TypeName<T>(), OldestVersion<T>()));
}
}
// Explicit form: migrate the file at `path` to T without keeping it open.
template<TableType T>
void Migrate(const std::filesystem::path& path, const OpenOptions& options = {}) {
MigrateTable<T>(path, TableFile::Inspect(path), options);
}
std::uint32_t PeekVersion(const std::filesystem::path& path) {
return TableFile::Inspect(path).header.schemaVersion;
}
}