Crafter.DB/interfaces/Crafter.DB-Storage.cppm

// SPDX-License-Identifier: LGPL-3.0-only
// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®

// The type-erased storage engine. A TableFile knows rows only as `rowSize`
// bytes and columns only through the descriptors the file carries, which is
// what lets it create, open, validate, recover, grow, compact and inspect any
// table without the C++ struct. Table<T> is a thin typed layer on top that
// computes column offsets at compile time and encodes/decodes through here.
export module Crafter.DB:Storage;
import std;
import :Error;
import :Format;
import :Formatter;
import :File;

export namespace Crafter::DB {
    struct TableStats {
        std::uint64_t slotCount = 0;     // rows including tombstones
        std::uint64_t liveCount = 0;     // rows not deleted
        std::uint64_t heapUsed = 0;      // bytes of variable-length data written
        std::uint64_t heapGarbage = 0;   // of which orphaned by Erase/Update; Compact() reclaims
        std::uint64_t fileLength = 0;
        std::uint64_t commitSeq = 0;
    };

    struct CreateParams {
        SchemaInfo schema;
        std::uint32_t schemaVersion = 0;
        std::uint64_t fingerprint = 0;
        std::uint16_t orderedColumn = noColumn;
        std::uint64_t initialRowBytes = 0;    // size the first extents for a known payload (Compact, Migrate)
        std::uint64_t initialHeapBytes = 0;
        std::uint32_t migratedFromVersion = 0;
    };

    // Everything Inspect() can tell about a table without its struct.
    struct Inspection {
        FileHeader header{};
        CommitRecord commit{};
        SchemaInfo schema;
    };

    // One scalar column value read through a descriptor, for tools and Verify.
    using ScalarValue = std::variant<std::int64_t, std::uint64_t, double>;

    // A type-erased view of one row: column access by name or index through
    // the file's own descriptors. This is what a dump tool or a migration from
    // a struct that no longer exists uses.
    class DynamicRow {
    public:
        DynamicRow() = default;
        DynamicRow(const std::byte* row, const SchemaInfo* schema, const Heap* heap) noexcept : row(row), schema(schema), heap(heap) {}

        bool Deleted() const noexcept;
        std::size_t ColumnCount() const noexcept { return schema->columns.size(); }
        const ColumnDescriptor& Column(std::size_t index) const noexcept { return schema->columns[index]; }
        std::string_view Name(std::size_t index) const noexcept { return schema->Name(index); }
        std::optional<std::size_t> Find(std::string_view name) const noexcept;
        // False only for the payload of a disengaged std::optional.
        bool Present(std::size_t index) const noexcept;
        // Integers, bools, enums and chrono reps as int64/uint64; floats as double.
        std::optional<ScalarValue> Scalar(std::size_t index) const noexcept;
        // String, Blob and FixedBytes columns as raw bytes (bounds-checked).
        std::optional<std::span<const std::byte>> Bytes(std::size_t index) const;
        std::optional<std::string_view> Text(std::size_t index) const;
        // "name=value, ..." for every column, for dumps and diagnostics.
        std::string Describe() const;

    private:
        const std::byte* row = nullptr;
        const SchemaInfo* schema = nullptr;
        const Heap* heap = nullptr;
    };

    struct VerifyResult {
        std::vector<std::string> problems;
        bool Ok() const noexcept { return problems.empty(); }
    };

    class TableFile {
    public:
        // Writes a complete empty table to a temp file beside `path`, syncs it
        // and renames it into place: a visible table file is always valid.
        static void Create(const std::filesystem::path& path, const CreateParams& params, const OpenOptions& options);
        static std::filesystem::path TempPath(const std::filesystem::path& path);
        static void Replace(const std::filesystem::path& path, const std::filesystem::path& temp);
        static std::filesystem::path BackupPath(const std::filesystem::path& path, std::uint32_t version);
        static Inspection Inspect(const std::filesystem::path& path);
        static void RemoveStaleTemps(const std::filesystem::path& path) noexcept;

        TableFile(const std::filesystem::path& path, const OpenOptions& options);
        TableFile(const TableFile&) = delete;
        TableFile& operator=(const TableFile&) = delete;
        ~TableFile();

        const FileHeader& Header() const noexcept { return header; }
        const SchemaInfo& Schema() const noexcept { return schema; }
        std::uint32_t RowSize() const noexcept { return header.rowSize; }
        bool Writable() const noexcept { return writable; }
        const std::filesystem::path& Path() const noexcept { return file.Path(); }
        void SetPath(std::filesystem::path path) { file.SetPath(std::move(path)); }
        TableStats Stats() const noexcept;

        // Published counts. Readers take SlotCount() once per scan; rows below
        // it are complete and never move.
        std::uint64_t SlotCount() const noexcept { return publishedRows.load(std::memory_order_acquire); }
        std::uint64_t LiveCount() const noexcept { return publishedLive.load(std::memory_order_relaxed); }
        std::uint64_t HeapUsed() const noexcept { return publishedHeap.load(std::memory_order_acquire); }
        // Set by the first Erase and never cleared while the file is open, so
        // readers decide between the dense scan and the tombstone-checking
        // scan from one monotonic flag rather than a racy pair of counters.
        bool HasTombstones() const noexcept { return tombstones.load(std::memory_order_acquire); }
        // False in a process that inherited this object through fork().
        bool OwnedByThisProcess() const noexcept;

        const std::byte* Rows() const noexcept { return rows.Base(); }
        const std::byte* Slot(std::uint64_t index) const noexcept { return rows.Base() + index * header.rowSize; }
        const Heap& HeapView() const noexcept { return heap; }
        static bool Deleted(const std::byte* row) noexcept;
        bool Deleted(std::uint64_t index) const noexcept { return Deleted(Slot(index)); }

        // Append protocol: BeginAppend(n) returns the first of n writable slots
        // (capacity guaranteed, contents undefined: callers encode into a
        // zeroed buffer and copy). Variable-length bytes go through Writer().
        // CommitAppend publishes; AbortAppend forgets the heap bytes.
        std::byte* BeginAppend(std::uint64_t count);
        HeapWriter& Writer() noexcept { return writer; }
        void CommitAppend(std::uint64_t count) noexcept;
        void AbortAppend() noexcept;

        std::byte* MutableSlot(std::uint64_t index);
        // Sets the tombstone bit. Returns false when the row was already deleted.
        bool MarkDeleted(std::uint64_t index);
        void AddHeapGarbage(std::uint64_t bytes) noexcept { heapGarbage += bytes; }
        // Out-of-line bytes a row references (for garbage accounting).
        std::uint64_t HeapBytesOf(const std::byte* row) const noexcept;

        // Durability: fdatasync the data, then write the inactive commit slot.
        void Flush(bool cleanClose = false);
        void Close();
        void RequireWritable() const;

        // Streams the live rows into `target` (same schema, empty, writable),
        // repacking their heap values in row order and dropping tombstones.
        void CompactInto(TableFile& target) const;

        DynamicRow Dynamic(std::uint64_t index) const noexcept { return { Slot(index), &schema, &heap }; }

        // Full consistency scan: every handle inside the committed heap, the
        // ordered column non-decreasing, live count equal to the recount,
        // presence bytes 0 or 1. Returns the problems instead of throwing so a
        // tool can print them all.
        VerifyResult Verify() const;

    private:
        class Arena final : public HeapWriter {
        public:
            void Attach(TableFile* table, std::byte* arenaBase, std::uint64_t arenaCursor, std::uint64_t arenaCapacity) noexcept {
                owner = table;
                base = arenaBase;
                cursor = arenaCursor;
                capacity = arenaCapacity;
            }
            void SetCapacity(std::uint64_t arenaCapacity) noexcept { capacity = arenaCapacity; }

        protected:
            void Grow(std::uint64_t neededCapacity) override;

        private:
            TableFile* owner = nullptr;
        };

        void AddExtent(std::uint32_t region, std::uint64_t minimumBytes);
        void EnsureHeapCapacity(std::uint64_t bytes);
        void WriteCommit(bool cleanClose);
        void RecountLive() noexcept;

        File file;
        OpenOptions options;
        bool writable = false;
        bool closed = false;
        FileHeader header{};
        SchemaInfo schema;
        CommitRecord current{};
        std::size_t activeSlot = 0;
        MappedPages headerPages;
        MappedRegion rows;
        MappedRegion heapRegion;
        Heap heap;
        Arena writer;
        std::atomic<std::uint64_t> publishedRows{ 0 };
        std::atomic<std::uint64_t> publishedLive{ 0 };
        std::atomic<std::uint64_t> publishedHeap{ 0 };
        std::atomic<bool> tombstones{ false };
        std::uint64_t heapGarbage = 0;
        std::uint64_t fileLength = 0;
        std::int64_t ownerProcess = 0;
        std::array<std::uint32_t, regionSlots> extentCount{};
        std::vector<std::size_t> varColumns;
    };
}