Initial commit: Crafter.DB
Header-only C++26 module library for typed, file-backed tables with schema reflection, migrations, queries, and crash-safe commits. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
commit
bfd0cc261b
GPG key ID:
2937E59CDCC1BCFB
34 changed files with 7506 additions and 0 deletions
201
interfaces/Crafter.DB-Storage.cppm
Normal file
201
interfaces/Crafter.DB-Storage.cppm
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
// SPDX-License-Identifier: LGPL-3.0-only
|
||||
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
||||
|
||||
// The type-erased storage engine. A TableFile knows rows only as `rowSize`
|
||||
// bytes and columns only through the descriptors the file carries, which is
|
||||
// what lets it create, open, validate, recover, grow, compact and inspect any
|
||||
// table without the C++ struct. Table<T> is a thin typed layer on top that
|
||||
// computes column offsets at compile time and encodes/decodes through here.
|
||||
export module Crafter.DB:Storage;
|
||||
import std;
|
||||
import :Error;
|
||||
import :Format;
|
||||
import :Formatter;
|
||||
import :File;
|
||||
|
||||
export namespace Crafter::DB {
|
||||
struct TableStats {
|
||||
std::uint64_t slotCount = 0; // rows including tombstones
|
||||
std::uint64_t liveCount = 0; // rows not deleted
|
||||
std::uint64_t heapUsed = 0; // bytes of variable-length data written
|
||||
std::uint64_t heapGarbage = 0; // of which orphaned by Erase/Update; Compact() reclaims
|
||||
std::uint64_t fileLength = 0;
|
||||
std::uint64_t commitSeq = 0;
|
||||
};
|
||||
|
||||
struct CreateParams {
|
||||
SchemaInfo schema;
|
||||
std::uint32_t schemaVersion = 0;
|
||||
std::uint64_t fingerprint = 0;
|
||||
std::uint16_t orderedColumn = noColumn;
|
||||
std::uint64_t initialRowBytes = 0; // size the first extents for a known payload (Compact, Migrate)
|
||||
std::uint64_t initialHeapBytes = 0;
|
||||
std::uint32_t migratedFromVersion = 0;
|
||||
};
|
||||
|
||||
// Everything Inspect() can tell about a table without its struct.
|
||||
struct Inspection {
|
||||
FileHeader header{};
|
||||
CommitRecord commit{};
|
||||
SchemaInfo schema;
|
||||
};
|
||||
|
||||
// One scalar column value read through a descriptor, for tools and Verify.
|
||||
using ScalarValue = std::variant<std::int64_t, std::uint64_t, double>;
|
||||
|
||||
// A type-erased view of one row: column access by name or index through
|
||||
// the file's own descriptors. This is what a dump tool or a migration from
|
||||
// a struct that no longer exists uses.
|
||||
class DynamicRow {
|
||||
public:
|
||||
DynamicRow() = default;
|
||||
DynamicRow(const std::byte* row, const SchemaInfo* schema, const Heap* heap) noexcept : row(row), schema(schema), heap(heap) {}
|
||||
|
||||
bool Deleted() const noexcept;
|
||||
std::size_t ColumnCount() const noexcept { return schema->columns.size(); }
|
||||
const ColumnDescriptor& Column(std::size_t index) const noexcept { return schema->columns[index]; }
|
||||
std::string_view Name(std::size_t index) const noexcept { return schema->Name(index); }
|
||||
std::optional<std::size_t> Find(std::string_view name) const noexcept;
|
||||
// False only for the payload of a disengaged std::optional.
|
||||
bool Present(std::size_t index) const noexcept;
|
||||
// Integers, bools, enums and chrono reps as int64/uint64; floats as double.
|
||||
std::optional<ScalarValue> Scalar(std::size_t index) const noexcept;
|
||||
// String, Blob and FixedBytes columns as raw bytes (bounds-checked).
|
||||
std::optional<std::span<const std::byte>> Bytes(std::size_t index) const;
|
||||
std::optional<std::string_view> Text(std::size_t index) const;
|
||||
// "name=value, ..." for every column, for dumps and diagnostics.
|
||||
std::string Describe() const;
|
||||
|
||||
private:
|
||||
const std::byte* row = nullptr;
|
||||
const SchemaInfo* schema = nullptr;
|
||||
const Heap* heap = nullptr;
|
||||
};
|
||||
|
||||
struct VerifyResult {
|
||||
std::vector<std::string> problems;
|
||||
bool Ok() const noexcept { return problems.empty(); }
|
||||
};
|
||||
|
||||
class TableFile {
|
||||
public:
|
||||
// Writes a complete empty table to a temp file beside `path`, syncs it
|
||||
// and renames it into place: a visible table file is always valid.
|
||||
static void Create(const std::filesystem::path& path, const CreateParams& params, const OpenOptions& options);
|
||||
static std::filesystem::path TempPath(const std::filesystem::path& path);
|
||||
static void Replace(const std::filesystem::path& path, const std::filesystem::path& temp);
|
||||
static std::filesystem::path BackupPath(const std::filesystem::path& path, std::uint32_t version);
|
||||
static Inspection Inspect(const std::filesystem::path& path);
|
||||
static void RemoveStaleTemps(const std::filesystem::path& path) noexcept;
|
||||
|
||||
TableFile(const std::filesystem::path& path, const OpenOptions& options);
|
||||
TableFile(const TableFile&) = delete;
|
||||
TableFile& operator=(const TableFile&) = delete;
|
||||
~TableFile();
|
||||
|
||||
const FileHeader& Header() const noexcept { return header; }
|
||||
const SchemaInfo& Schema() const noexcept { return schema; }
|
||||
std::uint32_t RowSize() const noexcept { return header.rowSize; }
|
||||
bool Writable() const noexcept { return writable; }
|
||||
const std::filesystem::path& Path() const noexcept { return file.Path(); }
|
||||
void SetPath(std::filesystem::path path) { file.SetPath(std::move(path)); }
|
||||
TableStats Stats() const noexcept;
|
||||
|
||||
// Published counts. Readers take SlotCount() once per scan; rows below
|
||||
// it are complete and never move.
|
||||
std::uint64_t SlotCount() const noexcept { return publishedRows.load(std::memory_order_acquire); }
|
||||
std::uint64_t LiveCount() const noexcept { return publishedLive.load(std::memory_order_relaxed); }
|
||||
std::uint64_t HeapUsed() const noexcept { return publishedHeap.load(std::memory_order_acquire); }
|
||||
// Set by the first Erase and never cleared while the file is open, so
|
||||
// readers decide between the dense scan and the tombstone-checking
|
||||
// scan from one monotonic flag rather than a racy pair of counters.
|
||||
bool HasTombstones() const noexcept { return tombstones.load(std::memory_order_acquire); }
|
||||
// False in a process that inherited this object through fork().
|
||||
bool OwnedByThisProcess() const noexcept;
|
||||
|
||||
const std::byte* Rows() const noexcept { return rows.Base(); }
|
||||
const std::byte* Slot(std::uint64_t index) const noexcept { return rows.Base() + index * header.rowSize; }
|
||||
const Heap& HeapView() const noexcept { return heap; }
|
||||
static bool Deleted(const std::byte* row) noexcept;
|
||||
bool Deleted(std::uint64_t index) const noexcept { return Deleted(Slot(index)); }
|
||||
|
||||
// Append protocol: BeginAppend(n) returns the first of n writable slots
|
||||
// (capacity guaranteed, contents undefined: callers encode into a
|
||||
// zeroed buffer and copy). Variable-length bytes go through Writer().
|
||||
// CommitAppend publishes; AbortAppend forgets the heap bytes.
|
||||
std::byte* BeginAppend(std::uint64_t count);
|
||||
HeapWriter& Writer() noexcept { return writer; }
|
||||
void CommitAppend(std::uint64_t count) noexcept;
|
||||
void AbortAppend() noexcept;
|
||||
|
||||
std::byte* MutableSlot(std::uint64_t index);
|
||||
// Sets the tombstone bit. Returns false when the row was already deleted.
|
||||
bool MarkDeleted(std::uint64_t index);
|
||||
void AddHeapGarbage(std::uint64_t bytes) noexcept { heapGarbage += bytes; }
|
||||
// Out-of-line bytes a row references (for garbage accounting).
|
||||
std::uint64_t HeapBytesOf(const std::byte* row) const noexcept;
|
||||
|
||||
// Durability: fdatasync the data, then write the inactive commit slot.
|
||||
void Flush(bool cleanClose = false);
|
||||
void Close();
|
||||
void RequireWritable() const;
|
||||
|
||||
// Streams the live rows into `target` (same schema, empty, writable),
|
||||
// repacking their heap values in row order and dropping tombstones.
|
||||
void CompactInto(TableFile& target) const;
|
||||
|
||||
DynamicRow Dynamic(std::uint64_t index) const noexcept { return { Slot(index), &schema, &heap }; }
|
||||
|
||||
// Full consistency scan: every handle inside the committed heap, the
|
||||
// ordered column non-decreasing, live count equal to the recount,
|
||||
// presence bytes 0 or 1. Returns the problems instead of throwing so a
|
||||
// tool can print them all.
|
||||
VerifyResult Verify() const;
|
||||
|
||||
private:
|
||||
class Arena final : public HeapWriter {
|
||||
public:
|
||||
void Attach(TableFile* table, std::byte* arenaBase, std::uint64_t arenaCursor, std::uint64_t arenaCapacity) noexcept {
|
||||
owner = table;
|
||||
base = arenaBase;
|
||||
cursor = arenaCursor;
|
||||
capacity = arenaCapacity;
|
||||
}
|
||||
void SetCapacity(std::uint64_t arenaCapacity) noexcept { capacity = arenaCapacity; }
|
||||
|
||||
protected:
|
||||
void Grow(std::uint64_t neededCapacity) override;
|
||||
|
||||
private:
|
||||
TableFile* owner = nullptr;
|
||||
};
|
||||
|
||||
void AddExtent(std::uint32_t region, std::uint64_t minimumBytes);
|
||||
void EnsureHeapCapacity(std::uint64_t bytes);
|
||||
void WriteCommit(bool cleanClose);
|
||||
void RecountLive() noexcept;
|
||||
|
||||
File file;
|
||||
OpenOptions options;
|
||||
bool writable = false;
|
||||
bool closed = false;
|
||||
FileHeader header{};
|
||||
SchemaInfo schema;
|
||||
CommitRecord current{};
|
||||
std::size_t activeSlot = 0;
|
||||
MappedPages headerPages;
|
||||
MappedRegion rows;
|
||||
MappedRegion heapRegion;
|
||||
Heap heap;
|
||||
Arena writer;
|
||||
std::atomic<std::uint64_t> publishedRows{ 0 };
|
||||
std::atomic<std::uint64_t> publishedLive{ 0 };
|
||||
std::atomic<std::uint64_t> publishedHeap{ 0 };
|
||||
std::atomic<bool> tombstones{ false };
|
||||
std::uint64_t heapGarbage = 0;
|
||||
std::uint64_t fileLength = 0;
|
||||
std::int64_t ownerProcess = 0;
|
||||
std::array<std::uint32_t, regionSlots> extentCount{};
|
||||
std::vector<std::size_t> varColumns;
|
||||
};
|
||||
}
|
||||
Loading…
Reference in a new issue