212 lines
11 KiB
Text
212 lines
11 KiB
Text
|
// SPDX-License-Identifier: LGPL-3.0-only
|
||
|
|
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||
|
|
|
||
|
|
// Schema migrations in C++. A table struct names its predecessor with
|
||
|
|
// `using Previous = OldStruct;` and either supplies
|
||
|
|
// `static T Migrate(OldStruct&&)` or, for purely additive changes, nothing:
|
||
|
|
// members are then copied by name and new ones value-initialised. Opening a
|
||
|
|
// file whose version is older than the struct's walks the chain to the type
|
||
|
|
// with that version and streams every live row through the conversions into
|
||
|
|
// a new file that replaces the old one atomically.
|
||
|
|
export module Crafter.DB:Migration;
|
||
|
|
import std;
|
||
|
|
import :Error;
|
||
|
|
import :Reflect;
|
||
|
|
import :Format;
|
||
|
|
import :Formatter;
|
||
|
|
import :Schema;
|
||
|
|
import :File;
|
||
|
|
import :Storage;
|
||
|
|
import :Codec;
|
||
|
|
|
||
|
|
export namespace Crafter::DB {
|
||
|
|
template<class From, std::size_t I, class To>
|
||
|
|
consteval std::size_t FindMemberByName() {
|
||
|
|
constexpr std::string_view name = MemberName<I, From>();
|
||
|
|
std::size_t found = noMember;
|
||
|
|
[&]<std::size_t... J>(std::index_sequence<J...>) {
|
||
|
|
((void)[&] { if (MemberName<J, To>() == name) found = J; }(), ...);
|
||
|
|
}(std::make_index_sequence<memberCount<To>>{});
|
||
|
|
return found;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Additive migration: every member of From must exist in To with the same
|
||
|
|
// name and type; members only in To are value-initialised.
|
||
|
|
template<Aggregate To, Aggregate From>
|
||
|
|
To CopyByName(From&& from) {
|
||
|
|
To to{};
|
||
|
|
[&]<std::size_t... I>(std::index_sequence<I...>) {
|
||
|
|
((void)[&] {
|
||
|
|
constexpr std::size_t j = FindMemberByName<From, I, To>();
|
||
|
|
static_assert(j != noMember, Message({ "Crafter.DB: member '", MemberName<I, From>(), "' of ", TypeName<From>(), " has no member of the same name in ", TypeName<To>(), "; a removed or renamed column needs an explicit `static ", TypeName<To>(), " Migrate(", TypeName<From>(), "&&)`." }));
|
||
|
|
if constexpr (j != noMember) {
|
||
|
|
static_assert(std::is_same_v<MemberType<I, From>, MemberType<j, To>>, Message({ "Crafter.DB: member '", MemberName<I, From>(), "' changed type between ", TypeName<From>(), " and ", TypeName<To>(), "; a type change needs an explicit `static ", TypeName<To>(), " Migrate(", TypeName<From>(), "&&)`." }));
|
||
|
|
Member<j>(to) = std::move(Member<I>(from));
|
||
|
|
}
|
||
|
|
}(), ...);
|
||
|
|
}(std::make_index_sequence<memberCount<From>>{});
|
||
|
|
return to;
|
||
|
|
}
|
||
|
|
|
||
|
|
// One step up the chain: Previous -> Next.
|
||
|
|
template<class Next>
|
||
|
|
Next ConvertStep(typename Next::Previous&& from) {
|
||
|
|
if constexpr (HasMigrate<Next>) return Next::Migrate(std::move(from));
|
||
|
|
else return CopyByName<Next>(std::move(from));
|
||
|
|
}
|
||
|
|
|
||
|
|
// The type directly above `From` in To's chain.
|
||
|
|
template<class To, class From>
|
||
|
|
consteval auto NextAbove() {
|
||
|
|
if constexpr (std::is_same_v<typename To::Previous, From>) return std::type_identity<To>{};
|
||
|
|
else return NextAbove<typename To::Previous, From>();
|
||
|
|
}
|
||
|
|
|
||
|
|
template<class To, class From>
|
||
|
|
To ConvertChain(From&& from) {
|
||
|
|
if constexpr (std::is_same_v<From, To>) return std::move(from);
|
||
|
|
else {
|
||
|
|
using Next = typename decltype(NextAbove<To, From>())::type;
|
||
|
|
Next next = ConvertStep<Next>(std::move(from));
|
||
|
|
return ConvertChain<To, Next>(std::move(next));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string DescribeColumns(const SchemaInfo& schema) {
|
||
|
|
std::string out = "[";
|
||
|
|
for (std::size_t i = 1; i < schema.columns.size(); i++) {
|
||
|
|
if (i != 1) out += ", ";
|
||
|
|
out += std::format("{}:{}", schema.Name(i), KindName(schema.columns[i].kind));
|
||
|
|
}
|
||
|
|
return out + "]";
|
||
|
|
}
|
||
|
|
|
||
|
|
template<Aggregate T>
|
||
|
|
void RequireFingerprint(const std::filesystem::path& path, const FileHeader& header, const SchemaInfo& schema) {
|
||
|
|
if (header.schemaFingerprint == Layout<T>::fingerprint && header.rowSize == Layout<T>::rowSize && header.orderedColumn == Layout<T>::orderedColumn) return;
|
||
|
|
const std::string onDisk = DescribeColumns(schema);
|
||
|
|
const std::string compiled = DescribeColumns(MakeSchemaInfo<T>());
|
||
|
|
if (onDisk == compiled && header.orderedColumn != Layout<T>::orderedColumn) {
|
||
|
|
const auto name = [&](std::uint16_t column) { return column == noColumn ? std::string("none") : std::string(schema.Name(column)); };
|
||
|
|
throw SchemaError(std::format("{}: {} version {} on disk is ordered by {} but the compiled struct is ordered by {}: `orderedBy` changed without bumping `version`", path.string(), TypeName<T>(), header.schemaVersion, name(header.orderedColumn), name(Layout<T>::orderedColumn)));
|
||
|
|
}
|
||
|
|
throw SchemaError(std::format("{}: {} version {} on disk has columns {} but the compiled struct has {}: the struct changed without bumping `version`", path.string(), TypeName<T>(), header.schemaVersion, onDisk, compiled));
|
||
|
|
}
|
||
|
|
|
||
|
|
// Writes the converted live rows of `old` into a fresh file and returns
|
||
|
|
// its path. Streaming keeps memory flat; `sorted` collects everything and
|
||
|
|
// stable-sorts by the new orderedBy first, for the case where the old data
|
||
|
|
// is not in the order the new struct declares.
|
||
|
|
template<TableType T, Aggregate Old>
|
||
|
|
std::filesystem::path WriteConverted(const TableFile& old, const std::filesystem::path& path, const OpenOptions& options, bool sorted) {
|
||
|
|
const TableStats stats = old.Stats();
|
||
|
|
CreateParams params;
|
||
|
|
params.schema = MakeSchemaInfo<T>();
|
||
|
|
params.schemaVersion = Layout<T>::version;
|
||
|
|
params.fingerprint = Layout<T>::fingerprint;
|
||
|
|
params.orderedColumn = Layout<T>::orderedColumn;
|
||
|
|
params.initialRowBytes = stats.liveCount * Layout<T>::rowSize;
|
||
|
|
params.initialHeapBytes = stats.heapUsed - std::min(stats.heapUsed, stats.heapGarbage);
|
||
|
|
params.migratedFromVersion = old.Header().schemaVersion;
|
||
|
|
OpenOptions targetOptions = options;
|
||
|
|
targetOptions.readOnly = false;
|
||
|
|
targetOptions.createDirectories = false;
|
||
|
|
const std::filesystem::path temp = TableFile::TempPath(path);
|
||
|
|
TableFile::Create(temp, params, targetOptions);
|
||
|
|
try {
|
||
|
|
TableFile target(temp, targetOptions);
|
||
|
|
const std::uint64_t slots = old.SlotCount();
|
||
|
|
if (sorted) {
|
||
|
|
std::vector<T> rows;
|
||
|
|
rows.reserve(stats.liveCount);
|
||
|
|
for (std::uint64_t i = 0; i < slots; i++) {
|
||
|
|
const std::byte* row = old.Slot(i);
|
||
|
|
if (!TableFile::Deleted(row)) rows.push_back(ConvertChain<T, Old>(DecodeRow<Old>(row, old.HeapView())));
|
||
|
|
}
|
||
|
|
if constexpr (HasOrderedBy<T>) std::ranges::stable_sort(rows, {}, [](const T& row) { return row.*T::orderedBy; });
|
||
|
|
AppendRows<T>(target, std::span<const T>(rows));
|
||
|
|
} else {
|
||
|
|
for (std::uint64_t i = 0; i < slots; i++) {
|
||
|
|
const std::byte* row = old.Slot(i);
|
||
|
|
if (TableFile::Deleted(row)) continue;
|
||
|
|
T converted = ConvertChain<T, Old>(DecodeRow<Old>(row, old.HeapView()));
|
||
|
|
AppendRows<T>(target, std::span<const T>(&converted, 1));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
target.Flush(true);
|
||
|
|
} catch (...) {
|
||
|
|
Os::Remove(temp);
|
||
|
|
throw;
|
||
|
|
}
|
||
|
|
return temp;
|
||
|
|
}
|
||
|
|
|
||
|
|
template<TableType T, Aggregate Old>
|
||
|
|
void RewriteFrom(const std::filesystem::path& path, const OpenOptions& options) {
|
||
|
|
OpenOptions sourceOptions = options;
|
||
|
|
sourceOptions.readOnly = false;
|
||
|
|
sourceOptions.mode = OpenMode::OpenExisting;
|
||
|
|
TableFile old(path, sourceOptions);
|
||
|
|
const std::uint32_t oldVersion = old.Header().schemaVersion;
|
||
|
|
|
||
|
|
std::filesystem::path temp;
|
||
|
|
try {
|
||
|
|
temp = WriteConverted<T, Old>(old, path, options, false);
|
||
|
|
} catch (const OrderError&) {
|
||
|
|
if constexpr (!HasOrderedBy<T>) throw;
|
||
|
|
temp = WriteConverted<T, Old>(old, path, options, true);
|
||
|
|
}
|
||
|
|
try {
|
||
|
|
if (options.backupBeforeMigrate) {
|
||
|
|
const std::filesystem::path backup = TableFile::BackupPath(path, oldVersion);
|
||
|
|
Os::Remove(backup);
|
||
|
|
Os::HardLink(path, backup);
|
||
|
|
}
|
||
|
|
old.Close();
|
||
|
|
TableFile::Replace(path, temp);
|
||
|
|
} catch (...) {
|
||
|
|
Os::Remove(temp);
|
||
|
|
throw;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
template<TableType T, Aggregate Old>
|
||
|
|
bool WalkChain(const std::filesystem::path& path, const Inspection& info, const OpenOptions& options) {
|
||
|
|
if (Layout<Old>::version == info.header.schemaVersion) {
|
||
|
|
RequireFingerprint<Old>(path, info.header, info.schema);
|
||
|
|
RewriteFrom<T, Old>(path, options);
|
||
|
|
return true;
|
||
|
|
}
|
||
|
|
if constexpr (HasPrevious<Old>) return WalkChain<T, typename Old::Previous>(path, info, options);
|
||
|
|
else return false;
|
||
|
|
}
|
||
|
|
|
||
|
|
template<class T>
|
||
|
|
consteval std::uint32_t OldestVersion() {
|
||
|
|
if constexpr (HasPrevious<T>) return OldestVersion<typename T::Previous>();
|
||
|
|
else return Layout<T>::version;
|
||
|
|
}
|
||
|
|
|
||
|
|
// Brings a file whose version differs from T's up to date, or throws.
|
||
|
|
template<TableType T>
|
||
|
|
void MigrateTable(const std::filesystem::path& path, const Inspection& info, const OpenOptions& options) {
|
||
|
|
const std::uint32_t fileVersion = info.header.schemaVersion;
|
||
|
|
if (fileVersion == Layout<T>::version) return;
|
||
|
|
if (fileVersion > Layout<T>::version) throw SchemaError(std::format("{}: file is at version {} but this build of {} is version {}; the file was written by a newer build", path.string(), fileVersion, TypeName<T>(), Layout<T>::version));
|
||
|
|
if (options.readOnly) throw MigrationError(std::format("{}: file is at version {} and {} is version {}, but the table was opened read-only", path.string(), fileVersion, TypeName<T>(), Layout<T>::version));
|
||
|
|
if (!options.migrate) throw MigrationError(std::format("{}: file is at version {} and {} is version {}, and migration was disabled", path.string(), fileVersion, TypeName<T>(), Layout<T>::version));
|
||
|
|
if (!WalkChain<T, T>(path, info, options)) {
|
||
|
|
throw SchemaError(std::format("{}: no migration path from on-disk version {} to {}: the Previous chain of {} stops at version {}", path.string(), fileVersion, Layout<T>::version, TypeName<T>(), OldestVersion<T>()));
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Explicit form: migrate the file at `path` to T without keeping it open.
|
||
|
|
template<TableType T>
|
||
|
|
void Migrate(const std::filesystem::path& path, const OpenOptions& options = {}) {
|
||
|
|
MigrateTable<T>(path, TableFile::Inspect(path), options);
|
||
|
|
}
|
||
|
|
|
||
|
|
std::uint32_t PeekVersion(const std::filesystem::path& path) {
|
||
|
|
return TableFile::Inspect(path).header.schemaVersion;
|
||
|
|
}
|
||
|
|
}
|