From eb720058f5fad213ec6678ed46ecc52a9c6bb1b3 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Sat, 31 Oct 2026 20:51:14 +0100 Subject: [PATCH 1/2] refactor: call the benchmark's comparison the legacy format The README, a comment in :Reflect and tests/Benchmark named the project whose transaction log this library replaces. That name means nothing to people using the library, so the benchmark's reference implementation is now the `Legacy` namespace, its column reads `legacy ms`, and the prose describes it as the hand-rolled iostream format Crafter.DB replaces. Timings and logic are unchanged; the suite still passes. Co-Authored-By: Claude Fable 5.1 --- README.md | 10 +++--- interfaces/Crafter.DB-Reflect.cppm | 2 +- tests/Benchmark/main.cpp | 54 +++++++++++++++--------------- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index e67b37f..e543be7 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Crafter.DB -An embedded, typed table store for C++26: a table is a plain struct, a file is a memory-mapped array of fixed-width rows, queries are C++. Built for the Catcrafts projects (it replaces Nostro's hand-rolled transaction log) and distributed as one C++ module, `Crafter.DB`. +An embedded, typed table store for C++26: a table is a plain struct, a file is a memory-mapped array of fixed-width rows, queries are C++. Built for the Catcrafts projects (it replaces a hand-rolled, iostream-based transaction log) and distributed as one C++ module, `Crafter.DB`. - **Structs are the schema.** Any aggregate with a `version` is a table. No macros, no registration, no code generation: member names, types and offsets are derived at compile time. - **Fixed-width rows, out-of-line strings.** Every row is `rowSize` bytes. Strings and blobs live in a heap region; the row holds a 16-byte handle that keeps values of up to 12 bytes, and the first 4 bytes of longer ones, inline, so `iban == x` is one 8-byte compare per row. @@ -194,18 +194,18 @@ One file per table: a 4 KiB header (magic, format and schema version, fingerprin ## Performance -`tests/Benchmark` compares Crafter.DB with a cleaned-up copy of Nostro's format (iostream, 2-byte length-prefixed strings) on 100 000 transactions, warm cache, one run on a Xeon w7-3465X: +`tests/Benchmark` compares Crafter.DB with a cleaned-up copy of the legacy format it replaces (iostream, 2-byte length-prefixed strings, linear scans) on 100 000 transactions, warm cache, one run on a Xeon w7-3465X: -| Operation | Nostro | Crafter.DB | Note | +| Operation | Legacy | Crafter.DB | Note | | --- | --- | --- | --- | -| append 100 000 rows | 16.5 ms | 11.1 ms | Crafter.DB includes `fdatasync`; Nostro syncs nothing | +| append 100 000 rows | 16.5 ms | 11.1 ms | Crafter.DB includes `fdatasync`; the legacy format syncs nothing | | last timestamp, 1000 × open + read | 1.9 ms | 30.0 ms | opening a mapped table costs one `mmap` per extent (about 20 for this size); open once per process | | load all, materialised | 36.0 ms | 10.4 ms | zero-copy scan of two columns: 0.8 ms | | time range, 10 % of rows | 10.0 ms | 0.9 ms | binary search + contiguous slice | | `iban == x` | 16.2 ms | 0.23–0.30 ms | 2–3 ns per row | | time range + `iban == x` | 11.2 ms | 0.03 ms | | -File size for that table: 17.2 MB against Nostro's 9.3 MB (fixed-width handles plus the slack of the last extents; `Compact()` right-sizes). +File size for that table: 17.2 MB against the legacy format's 9.3 MB (fixed-width handles plus the slack of the last extents; `Compact()` right-sizes). ## Building diff --git a/interfaces/Crafter.DB-Reflect.cppm b/interfaces/Crafter.DB-Reflect.cppm index 37626dd..11ba45e 100644 --- a/interfaces/Crafter.DB-Reflect.cppm +++ b/interfaces/Crafter.DB-Reflect.cppm @@ -72,7 +72,7 @@ export namespace Crafter::DB { return raw.substr(dot + 1, end - dot - 1); } - // The type's qualified name as the compiler spells it ("Nostro::Transaction"). + // The type's qualified name as the compiler spells it ("Ledger::Transaction"). // Used in diagnostics only, so the exact spelling is not load-bearing. template consteval std::string_view TypeName() { diff --git a/tests/Benchmark/main.cpp b/tests/Benchmark/main.cpp index d17f00e..c5a41e1 100644 --- a/tests/Benchmark/main.cpp +++ b/tests/Benchmark/main.cpp @@ -1,9 +1,9 @@ // SPDX-License-Identifier: LGPL-3.0-only // SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts® -// Crafter.DB against a cleaned-up copy of Nostro's hand-rolled format +// Crafter.DB against a cleaned-up copy of the legacy format it replaces // (iostream, uint16-prefixed strings inline, linear scans) on the five -// operations Nostro performs. Prints a table; never fails. Runs on 100k +// operations that log served. Prints a table; never fails. Runs on 100k // rows so a full suite stays fast; raise `rowCount` for a serious run. import Crafter.DB; @@ -49,8 +49,8 @@ namespace { return rows; } - // --- the Nostro format, cleaned up ----------------------------------------- - namespace Nostro { + // --- the legacy format, cleaned up ----------------------------------------- + namespace Legacy { void WriteString(std::ofstream& file, std::string_view str) { const std::uint16_t size = static_cast(str.size()); file.write(reinterpret_cast(&size), sizeof(size)); @@ -139,7 +139,7 @@ namespace { struct Line { std::string name; - double nostro; + double legacy; double crafter; std::string note; }; @@ -149,7 +149,7 @@ int main() { const fs::path dir = fs::temp_directory_path() / "crafter-db-benchmark"; fs::remove_all(dir); fs::create_directories(dir); - const fs::path nostroPath = dir / "nostro.bin"; + const fs::path legacyPath = dir / "legacy.bin"; const fs::path crafterPath = dir / "crafter.cdb"; const std::vector rows = MakeRows(); const std::string_view wantedIban = rows[rowCount / 2].iban; @@ -160,84 +160,84 @@ int main() { // append { - const double nostro = Millis([&] { Nostro::Save(nostroPath, rows); }); + const double legacy = Millis([&] { Legacy::Save(legacyPath, rows); }); const double crafter = Millis([&] { DB::Table table(crafterPath); table.Append(rows); table.Flush(); }); - lines.push_back({ std::format("append {} rows", rowCount), nostro, crafter, "incl. open, Crafter.DB incl. fdatasync" }); + lines.push_back({ std::format("append {} rows", rowCount), legacy, crafter, "incl. open, Crafter.DB incl. fdatasync" }); } // last timestamp (1000 opens each) { - const double nostro = Millis([&] { for (std::int32_t i = 0; i < 1000; i++) sink += Nostro::LastTime(nostroPath).time_since_epoch().count() & 1; }); + const double legacy = Millis([&] { for (std::int32_t i = 0; i < 1000; i++) sink += Legacy::LastTime(legacyPath).time_since_epoch().count() & 1; }); const double crafter = Millis([&] { for (std::int32_t i = 0; i < 1000; i++) { const DB::Table table(crafterPath, { .readOnly = true }); sink += table.Back()[&Transaction::timePoint].time_since_epoch().count() & 1; } }); - lines.push_back({ "last timestamp (1000x open+read)", nostro, crafter, "open dominates both" }); + lines.push_back({ "last timestamp (1000x open+read)", legacy, crafter, "open dominates both" }); } DB::Table table(crafterPath, { .readOnly = true }); // load all { - const double nostro = Millis([&] { sink += Nostro::LoadAll(nostroPath).size(); }); + const double legacy = Millis([&] { sink += Legacy::LoadAll(legacyPath).size(); }); const double crafter = Millis([&] { sink += table.Materialize().size(); }); double zeroCopy = Millis([&] { std::int64_t total = 0; for (DB::Row row : table) total += row[&Transaction::amount] + static_cast(row[&Transaction::iban].Length()); sink += static_cast(total & 1); }); - lines.push_back({ "load all (materialised)", nostro, crafter, std::format("zero-copy scan of two columns: {:.2f} ms", zeroCopy) }); + lines.push_back({ "load all (materialised)", legacy, crafter, std::format("zero-copy scan of two columns: {:.2f} ms", zeroCopy) }); } // time range (10% of rows) { - std::size_t nostroCount = 0; + std::size_t legacyCount = 0; std::size_t crafterCount = 0; - const double nostro = Millis([&] { nostroCount = Nostro::LoadRange(nostroPath, from, to).size(); }); + const double legacy = Millis([&] { legacyCount = Legacy::LoadRange(legacyPath, from, to).size(); }); const double crafter = Millis([&] { crafterCount = table.Range<&Transaction::timePoint>(from, to).Materialize().size(); }); - lines.push_back({ "time range (10% of rows, materialised)", nostro, crafter, std::format("{} / {} rows", nostroCount, crafterCount) }); + lines.push_back({ "time range (10% of rows, materialised)", legacy, crafter, std::format("{} / {} rows", legacyCount, crafterCount) }); } // iban == { - std::size_t nostroCount = 0; + std::size_t legacyCount = 0; std::size_t keyCount = 0; std::size_t exprCount = 0; std::size_t viewCount = 0; - const double nostro = Millis([&] { nostroCount = Nostro::LoadByIban(nostroPath, wantedIban).size(); }); + const double legacy = Millis([&] { legacyCount = Legacy::LoadByIban(legacyPath, wantedIban).size(); }); const double withKey = Millis([&] { const DB::Key key(wantedIban); keyCount = table.Where([&](DB::Row r) { return r[&Transaction::iban] == key; }).Count(); }); const double withExpression = Millis([&] { exprCount = table.Where(iban == wantedIban).Count(); }); const double withView = Millis([&] { viewCount = table.Where([&](DB::Row r) { return r[&Transaction::iban] == wantedIban; }).Count(); }); - lines.push_back({ "iban == x (lambda + Key)", nostro, withKey, std::format("{} / {} rows, {:.1f} ns/row", nostroCount, keyCount, withKey * 1e6 / rowCount) }); - lines.push_back({ "iban == x (expression)", nostro, withExpression, std::format("{} rows, {:.1f} ns/row", exprCount, withExpression * 1e6 / rowCount) }); - lines.push_back({ "iban == x (lambda + string_view)", nostro, withView, std::format("{} rows, {:.1f} ns/row", viewCount, withView * 1e6 / rowCount) }); + lines.push_back({ "iban == x (lambda + Key)", legacy, withKey, std::format("{} / {} rows, {:.1f} ns/row", legacyCount, keyCount, withKey * 1e6 / rowCount) }); + lines.push_back({ "iban == x (expression)", legacy, withExpression, std::format("{} rows, {:.1f} ns/row", exprCount, withExpression * 1e6 / rowCount) }); + lines.push_back({ "iban == x (lambda + string_view)", legacy, withView, std::format("{} rows, {:.1f} ns/row", viewCount, withView * 1e6 / rowCount) }); } // range + iban { - std::size_t nostroCount = 0; + std::size_t legacyCount = 0; std::size_t crafterCount = 0; - const double nostro = Millis([&] { - for (const Transaction& tr : Nostro::LoadRange(nostroPath, from, to)) nostroCount += tr.iban == wantedIban ? 1 : 0; + const double legacy = Millis([&] { + for (const Transaction& tr : Legacy::LoadRange(legacyPath, from, to)) legacyCount += tr.iban == wantedIban ? 1 : 0; }); const double crafter = Millis([&] { crafterCount = table.Where(timePoint >= from && timePoint <= to && iban == wantedIban).Count(); }); - lines.push_back({ "time range + iban == x", nostro, crafter, std::format("{} / {} rows", nostroCount, crafterCount) }); + lines.push_back({ "time range + iban == x", legacy, crafter, std::format("{} / {} rows", legacyCount, crafterCount) }); } - std::println("{:<42} {:>12} {:>12} {:>8} {}", "operation", "nostro ms", "crafter ms", "speedup", "note"); + std::println("{:<42} {:>12} {:>12} {:>8} {}", "operation", "legacy ms", "crafter ms", "speedup", "note"); for (const Line& line : lines) { - std::println("{:<42} {:>12.3f} {:>12.3f} {:>7.1f}x {}", line.name, line.nostro, line.crafter, line.nostro / std::max(line.crafter, 0.0001), line.note); + std::println("{:<42} {:>12.3f} {:>12.3f} {:>7.1f}x {}", line.name, line.legacy, line.crafter, line.legacy / std::max(line.crafter, 0.0001), line.note); } - std::println("files: nostro {} bytes, crafter {} bytes", fs::file_size(nostroPath), fs::file_size(crafterPath)); + std::println("files: legacy {} bytes, crafter {} bytes", fs::file_size(legacyPath), fs::file_size(crafterPath)); std::println("(sink {})", sink & 1); fs::remove_all(dir); From 38c9076e1df71d9beebc95deded65d827b802f3d Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Sat, 31 Oct 2026 20:51:14 +0100 Subject: [PATCH 2/2] docs: agent notes for this repository and for its consumers AGENTS.template.md is what crafter-build writes AGENTS.md from on every build and test run: the partition map, the toolchain facts the code leans on, build and test commands, test conventions, code style, the design rules with their reasons, commit guidelines and the gotchas found in the storage and file layers. AGENTS.consumer.md is the section crafter-build copies into the AGENTS.md of every project that depends on this library: declaring the dependency, the table-struct rules, locking and durability, writing and querying, migrations, an errors table and how to test a consumer. Both were fact-checked claim by claim against the interfaces, the implementation units and the tests. AGENTS.md is build output and is now gitignored, as in the sibling libraries. Co-Authored-By: Claude Fable 5.1 --- .gitignore | 2 + AGENTS.consumer.md | 86 ++++++++++++++++++++++++++ AGENTS.template.md | 150 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+) create mode 100644 AGENTS.consumer.md create mode 100644 AGENTS.template.md diff --git a/.gitignore b/.gitignore index d5697ec..e420a51 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ build/ bin/ +# written by every build and test run from AGENTS.template.md +/AGENTS.md diff --git a/AGENTS.consumer.md b/AGENTS.consumer.md new file mode 100644 index 0000000..c67bb6c --- /dev/null +++ b/AGENTS.consumer.md @@ -0,0 +1,86 @@ +## Crafter.DB + +The Catcrafts embedded table store: a C++26 module, `import Crafter.DB;`, in which a table is a plain struct with a `version`, a file is a memory-mapped array of fixed-width rows with an out-of-line heap for strings and blobs, reads are zero-copy views, queries are lambdas or column expressions that compose with `std::ranges`, and schema changes are C++ migrations run on open. Everything is in `namespace Crafter::DB` (the examples below assume `using namespace Crafter;`). Linux only: the engine uses `flock`, `fallocate` and fixed-address `mmap`. No dependencies, no link flags. Source and issues: https://forgejo.catcrafts.net/Catcrafts/Crafter.DB (LGPL-3.0-only). The README there is the manual, with the whole API and the file format; these notes are its `AGENTS.consumer.md`: a fix to them goes there, not into the copy crafter-build writes. + +### Building against it + +- `GitProject({ .source = { .url = "https://forgejo.catcrafts.net/Catcrafts/Crafter.DB.git" } })` (add `.commit` to pin), or `LocalProject({ .projectFile = "../Crafter.DB/project.cpp" })` for a sibling checkout, into `cfg.dependencies`. It is a static library whose partitions are nearly all templates: everything a `Table` does is instantiated in your translation units with your target and `--march`, and the only out-of-line code is the OS layer and the type-erased engine. It reads no project args and has no dependencies of its own, so a flag of yours that reaches it only changes its variant hash (a separate `bin/` directory per flag combination); strip your own flags from the `.args` you pass down if you would rather not, and pass the same list on every path that reaches it, or the build stops with two loads that disagree. +- Only Linux targets build it (`x86_64-unknown-linux-gnu` and `aarch64-alpine-linux-musl` have been built; the tests run on `*-linux-gnu`). A project that also targets `wasm32-*` or `x86_64-w64-mingw32` adds it to `cfg.dependencies` only when `cfg.target` contains `-linux-`, and keeps the code that uses it behind the same condition. The compiler is crafter-build's clang 23 with `import std;`; the library leans on C++26 features (structured-binding packs, pack indexing, generated `static_assert` messages) that no other toolchain in use here has. +- Nothing is defined for you to `#ifdef` on. Compile-time facts about a table come from `DB::Layout`: `rowSize`, `rowAlign`, `columnCount`, `fingerprint`, and `columns[i].kind`, `.offset` and `.name.View()` (the name is a fixed-capacity text, not a `std::string_view`). + +### Defining a table + +A table struct is an aggregate (public members, no base classes with members, no user-declared constructors), default constructible, with `static constexpr std::uint32_t version = 1;`. Optional statics: `static constexpr auto orderedBy = &T::member;` (an integer, enum, or integral-rep `duration` or `time_point` member; enforced on every write, enables binary search), `using Previous = OlderT;` and `static T Migrate(OlderT&&)` (see "Schema changes and migrations"). Every rule about the struct and its members is a `static_assert` whose message starts with `Crafter.DB:` and names them, so read the first error, not the instantiation stack. + +- Storable members: arithmetic types except `long double`, `bool`, `std::byte`, enums (stored as their underlying integer), `std::chrono::duration` and `time_point` (rep plus period, with the clock in the schema), `std::string`, `std::vector` for trivially copyable non-pointer `E`, `std::optional` of a storable `M` (not of another optional), `std::array` of scalars (one column per element) or of bytes (one fixed column), `DB::FixedString` (N bytes inline, NUL padded), nested aggregates of storable members (flattened to `outer.inner` columns), and anything with a `Crafter::DB::Formatter` specialisation. Pointers, references and member pointers are rejected. +- A struct with constructors is not an aggregate and is refused with a message that says to replace them with designated initialisers; `Table::Emplace` builds rows from fields, so the constructors are not missed. +- Columns are not laid out in declaration order: the flags byte sits at offset 0 and the leaves follow in ascending alignment, 2-byte columns before 8-byte ones. The README's seven-member `Transaction` is 88 bytes. A `std::string` or `std::vector` column is a 16-byte handle holding values of up to 12 bytes inline, and the first 4 bytes of longer ones (a vector of 8-byte-aligned elements such as `std::uint64_t` or `double` always goes to the heap), so equality against such a column is one 8-byte compare per row unless the lengths and prefixes match. Keys you filter on often are best kept at 12 bytes or under where the data allows (a BIC is, an IBAN is not); a code with a fixed maximum length is a `FixedString`. +- Your own types: specialise `Formatter` the way you would `std::formatter`. A fixed-width one declares `using Stored = ;` with `static Stored Encode(const T&)` and `static T Decode(const Stored&)`; a variable-length one inherits `Formatter` (or `Formatter>`) and overrides `static Var Encode(const T&, HeapWriter&)` and `static T Materialize(StringRef)`. A row still hands out the raw `StringRef` or `BlobRef` for a variable-length member; `Materialize()` is where your type comes back. +- Once the struct is settled, pin `DB::Layout::rowSize` and `DB::Layout::fingerprint` with a `static_assert` in a test: a failing assert is the compile-time form of the `SchemaError` your users would get at open, and tells you a `version` bump is due. + +### Opening and lifetime + +`DB::Table table(path);` creates the file when missing (directories `0700`, file `0600`), migrates it when older, and throws when it cannot. `DB::Table(path, { .mode = DB::OpenMode::OpenExisting, .readOnly = true })` opens a frozen reader. `OpenOptions` takes designated initialisers in this order: `mode` (`OpenOrCreate`, `OpenExisting`, `CreateNew`), `readOnly`, `createDirectories`, `filePermissions`, `directoryPermissions`, `durability` (`OnFlush`, `EveryAppend`), `migrate`, `backupBeforeMigrate`, `reserveBytes` (32 GiB), `excludeFromCoreDumps`, `prefetch`. + +- **Open once per process and keep the table open.** An open maps every extent of the file (one `mmap` each: 11 for the Benchmark's 100 000 rows appended in one batch, about 20 for the same rows appended one at a time) and a read-only open costs about 30 µs, which dominates any code that opens a table per query. A `Table` is move-only: hold it in a member or a `static`, not in each function that reads. +- **One writer per file at a time, across processes and within one.** A writer holds an exclusive `flock`, a read-only table a shared one: a second writable `Table` on the file, in another process or in this one, throws `LockedError`, a read-only table anywhere makes a writer's open fail, and a writer makes every reader's. Read-only tables coexist. Two processes that create the same table at once end up sharing one file. The lock is advisory: `cp` and other tools ignore it, and a copy taken while a writer is open reads back as of the writer's last commit record, which is its last `Flush`. +- **Durability is at `Flush()`, `Close()` and destruction, not at `Append`.** Under `Durability::OnFlush` (the default) an appended row is visible to this process at once and reaches the disk at the next `Flush`; a crash loses what was not flushed and never a committed row. An `Update` or `Set` that writes heap bytes (a string longer than 12 bytes, a blob) flushes on its own, an `fdatasync` each, and so commits every row appended before it. `Durability::EveryAppend` flushes after every write. The destructor flushes but cannot report; call `Close()` where a failed final flush must throw. After `fork()` the child may read the inherited table; every write and every explicit `Flush()` in it throws `Error`, while `Close()` and the destructor skip the flush silently, so the child's stale counters never reach the file. +- **Files beside the table**: `.tmp-<12 hex>` while the table is being created and while `Compact`, `Clear` or a migration is writing (stale ones go on the next writable open), and `.v.bak` after a migration (a hard link of the pre-migration file; delete it when it has served). The directory must stay writable for those three operations. Use a directory per table or distinct basenames; `.cdb` is the convention. +- **Address space.** Each open table reserves `reserveBytes` twice (rows and heap), 64 GiB of virtual space by default, none of it committed. Under a `ulimit -v`, or with hundreds of tables open, lower `reserveBytes` to a few times the table's size; a reservation that fills during an append throws `IOError` naming the option. +- A table created on a 4 KiB-page kernel cannot be opened on a 16 KiB-page one (`IOError`, `not_supported`); the extent granule is recorded in the header. + +### Writing + +`Append(row)` and `Append(range)` return the `RowId` of the first new row (ids are slot indices from 0, stable until `Compact` or `Clear`); a range is one reservation and one publish, all or nothing. `Emplace(args...)` takes exactly one argument per member in declaration order, converted implicitly (a `std::string_view` is enough for a string member; an `int` for an enum member does not compile). `Update(id, row)` replaces a live row; `Set<&T::member>(id, value)` one member; `Erase(id)` tombstones (false when already erased); `EraseIf(pred)` returns the count; `Flush()`; `Compact()` rewrites without tombstones and heap garbage; `Clear()` empties. + +- On an `orderedBy` table a write whose key would sort before the previous row (or, for `Update` and `Set`, after the next) throws `OrderError` before anything is written; equal keys are allowed; an unsorted batch is refused whole. Rows must arrive in key order: sort before `Append`, or leave `orderedBy` out and sort when reading. +- `Update`, `Set` and `Erase` leave heap garbage (the old string bytes); `Stats().heapGarbage` says how much, and only `Compact()` reclaims it. `Compact` and `Clear` invalidate every `RowId`, `Row` and `StringRef` taken before them; `Update` and `Set` invalidate the views of that row; `Append` and `Erase` invalidate nothing. +- `Update` and `Set` never leave a row pointing at heap bytes that are not on disk (those are committed first), but the row's fixed-width columns are not written atomically: a crash or power loss mid-row leaves the old row, the new one, or a mix of the two. A log of appends is the shape the library is built for. +- Limits: a value below 4 GiB (`Error`), a row of at most 65535 bytes and 65535 columns (compile time), a region of at most 32 extents, roughly 17 GiB (`IOError` saying to `Compact()`), and the disk (`fallocate` fails with `IOError` before any page is touched, never a `SIGBUS`). + +### Reading and querying + +`table.Size()` is the live rows, `SlotCount()` the id space including tombstones, `table[id]` an unchecked `Row` view (16 bytes, no allocation), `At(id)` a checked one, `Contains(id)`, `Front()`, `Back()` (skips trailing tombstones). `row[&T::member]` and `row.Get<&T::member>()` read one column from the mapping: the value for scalars, chrono and `FixedString`; a `StringRef` for a string (converts to `std::string_view`, compares with `==` to literals, `std::string_view`, `std::string`, `Key` and other refs without building a view, orders with `<=>`); a `BlobRef` for a vector (converts to `std::span`); an `std::optional` of the view for an optional. `row.Materialize()` (also `explicit operator T`) copies the row out; `table.Materialize()` every live row. `std::format("{}", row)` prints `Type{member=value, …}`. + +- `for (DB::Row row : table)` is the live rows in id order and composes with `std::views`; `table.Slots()` is random access over every slot with `row.Erased()`; `table.Rows(first, last)` a slot range; `table.Select<&T::member>()` a view of one column. +- `table.Where(pred)` is lazy and returns a chainable `Query`: `Materialize()`, `Count()`, `First()` (an `std::optional>`), `Any()`, `Where()`, `Select<&T::m>()`. `pred` is a lambda over `Row` or a column expression built from tags, `inline constexpr DB::Column<&T::iban> iban{};` and then `iban == x && amount > 100'00`, with `== != < <= > >= && || !`. Operands are copied into the expression, so a temporary `std::string` is fine. +- **Encode the key once.** In a lambda, compare against a `const DB::Key key(text);` built outside the loop rather than a `std::string_view` (correct, but the encoding is redone per row); a column expression does it for you. The predicate is a template parameter and inlines into the scan: no `std::function`, no per-row call. Measured on 100 000 rows (Xeon w7-3465X, warm cache): `iban == x` at 2 to 5 ns per row, a time range covering 10 % of the rows about 1 ms materialised, every row materialised 10 to 13 ms. +- **The ordered column.** `Range<&T::key>(from, to)` (inclusive), `From`, `After`, `Until`, `Before`, `LowerBound` and `UpperBound` are binary searches and compile only for the member `orderedBy` names (a `static_assert` says so for any other). `table.Range<…>(a, b).Where(iban == x)` filters inside the slice, and `table.Where(timePoint >= a && timePoint <= b && iban == x)` does the same: bounds on the ordered column in a top-level `&&` chain become the search. Under `||` or `!` nothing is extracted and the expression runs as a full scan, still correct. Tombstoned rows keep their key, so searches stay valid until `Compact()`. `std::ranges::lower_bound(table.Slots(), key, {}, [](DB::Row r) { return r[&T::m]; })` is the escape hatch for any other column. +- A `Query` over a plain range (`Rows`, `Range`, `From` and the like) can be iterated as a `const` object; one built with `Where` cannot (`std::ranges::filter_view`), so hold it in a non-const local. `Select()` on a `Query` is rvalue-only: call it on the temporary, or `std::move` the query. +- Threads: writes come from one thread at a time. While it runs `Append`, `Emplace` or `Erase`, other threads may read (a scan that started before an `Erase` may still yield the row); `Update`, `Set`, `Compact` and `Clear` need exclusive access from you. Views stay valid across `Append` and `Erase`. + +### Schema changes and migrations + +The file carries the struct's `version` and a fingerprint of the layout: column names, kinds, offsets, sizes, flags, chrono period and clock, blob element type and the ordered column. Any change to those without a `version` bump is refused at open with a `SchemaError` listing both column sets. That includes renaming a member, changing a type or an enum's underlying type, reordering members of the same alignment class, adding or changing `orderedBy`, and changing a `Formatter`'s `Stored`. + +- Bump `version`, keep the old struct under another name (`TransactionV1`), and point `using Previous = TransactionV1;` at it. Added members need nothing more: old rows are copied by member name and new members value-initialised. A removed, renamed or retyped member needs `static T Migrate(Previous&&)`; without it the build fails naming the member. Chains are walked (`V1 → V2 → V3`) and the versions along `Previous` must strictly decrease. Keep every old struct for as long as files of that version may exist. +- The migration runs in the constructor on the first open with the newer code: every live row is streamed through the chain into a new file that replaces the old one atomically, and the old one is kept as `.v.bak` (`backupBeforeMigrate`). Rows that do not satisfy a new `orderedBy` are stable-sorted by it. A large table makes that first open take a while, once. +- Refusals: a file newer than the code, or one the chain does not reach, is a `SchemaError`; an older file opened `readOnly` or with `migrate = false` is a `MigrationError`. `DB::PeekVersion(path)` reads the version from the header alone, without the lock; `DB::Migrate(path)` migrates without keeping the table open; `DB::TableFile::Inspect(path)` returns header, commit and schema (an `Inspection`) without the struct and without the lock; and `DB::DynamicRow` reads columns by name from any table, for dumps and tools. +- The file format itself has a `formatVersion` (1) that must match exactly; a bump there is a library release note, not something a consumer handles. + +### Errors + +Everything the library throws derives from `Crafter::DB::Error`, a `std::runtime_error`, except where noted. Most messages start with the file path; the 4 GiB `Error`, the heap-handle `CorruptError` and the address-space and `msync` `IOError`s carry none. + +| Call | On failure | +|---|---| +| `Table` constructor, `Migrate` | `IOError` (with `code` and `path`: a missing file under `OpenExisting`, an existing one under `CreateNew`, permissions, page-size mismatch, address space), `CorruptError` (not a table, unsupported format, both commit records damaged, extents or schema out of bounds), `SchemaError`, `MigrationError`, `LockedError` | +| `PeekVersion`, `TableFile::Inspect` | `IOError` (missing file, permissions), `CorruptError`; neither takes the lock | +| `Append`, `Emplace`, `Update`, `Set`, `Erase`, `EraseIf`, `Compact`, `Clear` | `ReadOnlyError`; `Error` on a closed table or one inherited through `fork`; `OrderError` before anything is written; `IOError` when the disk, the extent table or the reservation is full; `Error` for a value of 4 GiB or more; `std::length_error` for a `FixedString` that does not fit; `CorruptError` from `Compact` on a damaged heap handle | +| `Flush`, `Close` | `IOError` from `fdatasync` or `msync`; `Flush` throws `Error` in a forked child, where `Close` skips the flush; `Flush` is a no-op on a read-only table, `Close` still closes it and releases the lock | +| `Update`, `Set`, `Erase`, `At` with an id outside the table, or (`Update`, `Set`, `At`) an erased one; `Front`, `Back` on an empty table | `std::out_of_range` (`operator[]` does not check) | +| `StringRef::View`, `BlobRef::View`, `Materialize`, any read of a long value | `CorruptError` when the handle points outside the committed heap, which is a damaged file | +| `Verify()` | never throws for content; returns a `VerifyResult` listing the problems, at most a thousand | +| API misuse: a non-aggregate or `version`-less struct, an unstorable member, `orderedBy` on a float or string, `Range` on another member, a wrong `Emplace` argument, a `Previous` member with no counterpart | a compile error whose `static_assert` message starts with `Crafter.DB:` | + +### Testing a consumer + +- Work under `fs::temp_directory_path() / "-"`, `remove_all` first and last. crafter-build runs tests in parallel, so each test gets a directory of its own. +- Destroy or `Close()` a writable `Table` before opening the same file again, and a read-only one before opening it writable: the lock is held until then, and the conflicting open throws `LockedError` in the same process. +- A crash test is a `fork()`ed child that writes and ends with `_exit(0)` while its table is still in scope, so no destructor flushes; the parent then opens the file and sees exactly the flushed rows. `tests/ShouldRecoverCommits/main.cpp` in the Crafter.DB repository has the helper. +- End a test that exercised writes with `table.Verify().Ok()`, and pin `DB::Layout::rowSize` and `fingerprint` with `static_assert`s so an accidental schema change fails the build rather than a user's open. +- `std::format("{}", row)` and `DB::TableFile::Inspect(path)` are the debugging tools; `table.File().Dynamic(id).Describe()` prints any row of any table by column name. + +### Names to get right + +`Materialize` (with a z), `Emplace`, `Erase` and `EraseIf` (not Delete or Remove), `Compact`, `Verify`, `Stats`, `Flush`, `Close`; `Formatter` is the customisation point even though it encodes bytes, not text; the statics are `version` and `orderedBy` (camelCase) and `Previous` and `Migrate` (PascalCase); `RowId` is a `std::uint64_t`; `Key` is the pre-encoded search key, `StringRef` and `BlobRef` the column views, `Column<&T::m>` the tag type. diff --git a/AGENTS.template.md b/AGENTS.template.md new file mode 100644 index 0000000..a505b94 --- /dev/null +++ b/AGENTS.template.md @@ -0,0 +1,150 @@ +# AGENTS.md + +Guidance for coding agents and contributors working on Crafter.DB itself. The [README](README.md) is the user-facing manual, and [AGENTS.consumer.md](AGENTS.consumer.md) is what agents working in a *consumer* of this library read: crafter-build copies it into the dependency-notes section of the consumer's `AGENTS.md` on every build. When any of the three disagrees with the code, the code wins; the interfaces under `interfaces/` are the contract, and `tests/ShouldKeepLayoutGolden/main.cpp` is the contract for the bytes on disk. + +## Project overview + +Crafter.DB is the Catcrafts embedded table store: one C++26 module, `Crafter.DB`, with twelve partitions behind the umbrella (thirteen interface files) and two implementation units, no dependencies, Linux only. A table is a plain struct with a `version`; a file is a memory-mapped array of fixed-width rows plus a heap for strings and blobs; reads are zero-copy views; queries are lambdas or column expressions; schema changes are C++ migrations run on open. It replaces a hand-rolled, iostream-based transaction log in the first Catcrafts project that needed one (that port is still to do) and serves any Crafter project that keeps rows in a file. + +| Path | What lives there | +|---|---| +| `interfaces/Crafter.DB.cppm` | The umbrella: `export import`s every partition. | +| `interfaces/Crafter.DB-.cppm` | One partition per file, each but `:Error` opening with a comment that says what the partition owns. All but `:File`, `:Storage` and two plain functions in `:Migration` (`DescribeColumns`, `PeekVersion`) are templates and inline code. | +| `implementations/Crafter.DB-File.cpp`, `-Storage.cpp` | The two implementation units, `Crafter.DB:File_impl` and `Crafter.DB:Storage_impl`. `File` is the only unit that includes a C header. | +| `project.cpp` | The build description: the static library `Crafter.DB` and, for `*-linux-gnu` targets, the fourteen tests. No dependencies, no lint rules. | +| `tests//main.cpp` | One test program per directory, declared in `project.cpp`. | +| `AGENTS.template.md`, `AGENTS.consumer.md` | The source of this file, and the notes for consumers. Every build and test run writes `AGENTS.md` from the template; the library has no dependencies, so the dependency-notes section never appears here. `AGENTS.md` is gitignored: edit the template. | +| `bin/`, `build/` | Generated and gitignored. `bin/Crafter.DB----/` holds the PCMs, `std.pcm` and `libCrafter.DB.a`, `build//` the objects, and each test has a pair of its own. `build/test-logs/.log` keeps the output of failed tests. | + +Partitions, from the bottom up (each imports only the ones above it): + +- `:Error`: `Error : std::runtime_error` and its subclasses `IOError` (with `code` and `path`), `CorruptError`, `SchemaError`, `MigrationError`, `LockedError`, `OrderError`, `ReadOnlyError`. Everything the library throws derives from `Error`, except the `std::out_of_range` of `At()` and friends and the `std::length_error` of a `FixedString` that does not fit. +- `:Reflect`: compile-time introspection of aggregates without P2996: `memberCount`, `Member`, `MemberType`, `MemberName()` (parsed out of `std::source_location::function_name()`), `MemberIndex()`, `MemberIndexOf`, `TypeName()`, `ForEachMember`, the `Aggregate` concept. The `static_assert`s at the bottom fail the library's own build if a compiler changes the shape of `function_name()`. +- `:Format`: the on-disk PODs (`FileHeader`, `CommitRecord`, `Extent`, `SchemaBlobHeader`, `ColumnDescriptor`, `Var`), `Kind`, the column flags, FNV-1a, CRC-32C, `SchemaInfo` (serialise, parse with range checks, fingerprint). Every struct size and the load-bearing offsets are pinned by `static_assert`s. +- `:Formatter`: `Formatter`, the customisation point, with the `Scalar`, `FixedFormatter` and `VarFormatter` concepts; `Heap` and `HeapWriter`; `EncodeVar`/`VarBytes`; `Key`, `StringRef`, `BlobRef`, `FixedString`; the built-in formatters for `std::string`, `std::vector`, `std::chrono::duration` and `time_point`. +- `:Schema`: `Layout`, computed `consteval`: members classified by `Shape`, nested aggregates flattened to dotted names, leaves placed by ascending alignment after the flags byte, the fingerprint; `TableType`, `HasOrderedBy`, `HasPrevious`, `HasMigrate`, `OrderableKey`; `CheckTable()` with the diagnostics for the struct and its members (the size limits sit in `Layout` and `Arrange`); `MakeSchemaInfo()`; the `SchemaTests` namespace of compile-time checks. +- `:File`: everything that talks to the OS: `OpenOptions`, `OpenMode`, `Durability`; `File` (descriptors, `flock`, `fallocate` with the `posix_fallocate` fallback, `pread`/`pwrite`, `fdatasync`/`fsync`); `MappedRegion` (a `PROT_NONE` reservation filled with `MAP_FIXED` extents); `MappedPages`; the `Os` namespace (private `mkdir -p`, rename without replace, hard link, directory sync, random bytes, page size, pid). Nothing here knows what a table is. +- `:Storage`: the type-erased engine. `TableFile` creates, opens, validates, recovers, grows, flushes, compacts and verifies a table knowing rows only as `rowSize` bytes and columns only through the file's descriptors; `Inspection` and `TableFile::Inspect`, `DynamicRow`, `TableStats`, `VerifyResult`, `CreateParams`. +- `:Codec`: typed encode/decode over `Layout`: `EncodeRow`, `EmplaceRow`, `EncodeMember`, `DecodeRow`, `DecodeMember`, `View`, `CopyLeaves`/`ZeroLeaves`, the ordered-column helpers (`LowerBoundSlot`, `UpperBoundSlot`, `CheckOrder`) and `AppendRows`. +- `:Row`: `RowId`, `Column`, `Row` (the 16-byte view), `LiveIterator`/`LiveRange` (sentinel-terminated, skips tombstones), `SlotView` (random access over every slot), `DescribeRow`, `std::formatter>` and `std::formatter`. +- `:Query`: the expression types `Compare`, `And`, `Or`, `Not` and their operators on `Column` tags, `Bounds` and `CollectBounds` (bound extraction from a top-level `&&`), the chainable `Query`. +- `:Migration`: `CopyByName`, `ConvertChain`, `RequireFingerprint`, `WriteConverted`/`RewriteFrom`/`WalkChain`, `MigrateTable`, and the exported `Migrate(path)` and `PeekVersion(path)`. +- `:Table`: `Table`, the user-facing handle: open/create/migrate, every write, iteration, `Where`/`Select`, the ordered-column searches, and `Compact`/`Clear` through `Rebuild`. + +Numbers the code and the tests pin (`tests/ShouldKeepLayoutGolden`, the `SchemaTests` namespace): the README's `Transaction` is 88 bytes, 8-aligned, with `currency` at offset 2, `name` at 8, `bic` at 24, `iban` at 40, `description` at 56, `amount` at 72 and `timePoint` at 80; a `Var` handle is 16 bytes with 12 inline; the header is one 4 KiB page followed by at least one schema page; the first extent of each region is 16 KiB, extents double up to 1 GiB, and a region holds at most 32 of them (roughly 17 GiB); 32 GiB of address space is reserved per region; a row is at most 65535 bytes and a table at most 65535 columns; a value is below 4 GiB. A fresh table is 24 KiB without string or blob columns and 40 KiB with. + +## Dev environment + +crafter-build is the only tool, from its Flatpak (`alias crafter-build='flatpak run net.catcrafts.CrafterBuild'`; this machine has its `stable` and `beta` branches, both 4.0.0, and a build installed from a Crafter.Build checkout is branch `local`, chosen with `--branch=`). It compiles with its bundled clang 23.1.2 and libc++ 23.1.2, `-std=c++26`, `import std;`. The host's `/usr/bin/clang++` is not used and is not a supported compiler for this library. The project has no dependencies, so nothing is fetched and no network is needed. The host is Linux x86_64. + +What that compiler does and does not have, checked on 2026-10-17 (the library leans on all of it): + +- No P2996 reflection: `-freflection` is accepted and ignored, and there is no ``. `:Reflect` exists because of that and is the only partition to replace when a clang ships `std::meta`. +- Structured-binding packs (`auto&& [...m] = t;`), pack indexing (`m...[I]`), user-generated `static_assert` messages, deducing `this`, `[[clang::no_destroy]]`, `std::atomic_ref`, `std::ranges::fold_left`, `std::format` and `std::println`: all in use. +- libc++ 23 lacks `std::start_lifetime_as`, `std::inplace_vector` and `std::simd`, so row fields are read with constant-size `memcpy` (which folds to a load) and SIMD scans are not attempted. +- libc++'s `system_clock::duration` is microseconds (libstdc++'s is nanoseconds). The chrono period is part of the fingerprint for that reason, and the golden test pins `paramDen == 1000000`. + +The Flatpak sandbox sees the host filesystem but has its own `/tmp` (`fs::temp_directory_path()` inside a test is a per-app directory under `$XDG_RUNTIME_DIR/.flatpak/net.catcrafts.CrafterBuild/tmp`, shared by instances running at the same time and gone once none is), keeps its caches under `~/.var/app/net.catcrafts.CrafterBuild/cache/crafter.build/`, and a binary it built runs only inside it: `flatpak run --command=$PWD/bin/-/ net.catcrafts.CrafterBuild`. That is how you see a passing test's output, which the runner does not keep, the Benchmark's table in particular. `touch tests//main.cpp && crafter-build -v test ` prints the exact compiler command, the model for a scratch program against the built PCMs. + +`std.pcm` is built once per target, `--march` and compiler into `~/.var/app/net.catcrafts.CrafterBuild/cache/crafter.build/---/` and copied into each variant's directory; a toolchain update rebuilds it once, and a warm run of the whole suite takes about 2.5 seconds. The variant hash includes the compiler, so the directories of earlier compilers, and of the stub this repository started as (`bin/Crafter.DB-…-97c53ceb1aa99864/`, with colon-named PCMs), stay until `crafter-build clean`; glob for a directory rather than predicting its hash. + +## Build, test and style commands + +```bash +crafter-build # bin/Crafter.DB-/libCrafter.DB.a and the PCMs +crafter-build test # build and run the fourteen tests, Benchmark included +crafter-build test 'ShouldM*' # name glob +crafter-build test --list +crafter-build test --jobs=1 --timeout=120 +crafter-build -v test ShouldQuery # with the compiler commands +crafter-build clean # delete bin/ and build/ +``` + +`project.cpp` registers no lint rules and does not pull in Crafter.LintRules (a `#pragma crafter dependency` would `git pull` and rebuild it on every load of `project.cpp`), so `crafter-build lint` and `crafter-build format` exit 1 with "No lint rules defined." Style is checked by eye, see "Code style". There is no CI. Before a change is done, `crafter-build test` passes: 14 passed as of 2026-10-31. Failed, crashed and timed-out tests write `build/test-logs/.log`; a later passing run does not delete the file (three from the first day are still there), so check its timestamp before trusting it. + +## Testing instructions + +Tests are declared, not discovered: one `cfg.AddTest("").Dependencies({ &cfg });` line in `project.cpp`, inside the `if (cfg.target.ends_with("-linux-gnu"))` block (the engine uses `flock`, `fallocate` and fixed-address `mmap`; the musl target gets the library and no tests). The source is `tests//main.cpp`, an `int main()` linked against the local `libCrafter.DB.a`: exit 0 passes, nonzero fails, 77 skips (unused here). The runner gives each test 60 s in its own process, in parallel. + +Conventions the fourteen follow: + +- Names are `Should`; `Benchmark` is the exception. Open `main.cpp` with a comment that states the behaviour under test and the failure it guards against; `tests/ShouldRecoverCommits/main.cpp` and `tests/ShouldReadWhileAppending/main.cpp` show the shape, and a regression test names the bug. +- `Check(condition, what)` and `CheckEqual(actual, expected, what)` in an anonymous namespace print `FAIL: …` (with both values for `CheckEqual`) and count; `main` prints `ok` and returns 0 only when nothing failed. Do not stop at the first failure unless later steps are meaningless without it. +- Each test works under `fs::temp_directory_path() / "crafter-db-"`, `remove_all`ed first and last, with a name of its own because the runner is parallel. Table structs are declared per test: the README's `Transaction` where the reference layout matters, a one- or two-member struct where it does not (`Item`, `Entry`, `Tick`). +- Process-level behaviour is tested with `fork()`. `ShouldLockAcrossProcesses` opens the table in a child and maps the outcome to an exit code; `ShouldRecoverCommits` runs the writer in a child that ends with `_exit(0)` while its `Table` is still in scope, so no destructor and no `Flush` runs, which is the only honest way to simulate a crash. Those two `#include ` and `` before their imports. Damage to a file is done with a `Patch(path, offset, bytes)` helper over `std::fstream` in `ShouldRecoverCommits`, `ShouldRejectSchemaChange` and `ShouldVerify`. +- Concurrency is tested with `std::jthread` readers scanning while the main thread appends and erases (`ShouldReadWhileAppending`), and such a run ends with `table.Verify().Ok()`. +- `Benchmark` never fails. It prints the legacy format (a cleaned-up copy of the hand-rolled iostream log Crafter.DB replaces) against Crafter.DB on 100 000 rows with a warm cache; raise `rowCount` for a serious run, and quote its numbers with the machine (the README's are a Xeon w7-3465X). + +What is covered: layout and fingerprint (`ShouldKeepLayoutGolden`), encode/decode and reopen (`ShouldRoundTrip`), pointer stability across extents (`ShouldKeepViewsStableAcrossGrowth`), `StringRef`/`BlobRef` semantics (`ShouldCompareStringRefs`), every query form and the ordered column (`ShouldQuery`), tombstones and `Compact` (`ShouldEraseAndCompact`), every member shape and both formatter shapes (`ShouldFormatCustomTypes`), the migration chain and its refusals (`ShouldMigrate`), schema and corruption refusals (`ShouldRejectSchemaChange`), the commit protocol under crashes (`ShouldRecoverCommits`), locking (`ShouldLockAcrossProcesses`), `Verify` and `DynamicRow` (`ShouldVerify`), readers against a writer (`ShouldReadWhileAppending`). Not covered: a table on a 16 KiB-page kernel, `reserveBytes` exhaustion, the `posix_fallocate` fallback, `prefetch`. + +A bug fix ships with a test that fails before it and passes after. A change to the bytes on disk makes `ShouldKeepLayoutGolden` fail: that is the point of it, so read design rule 4 before updating its expectations. + +## Code style + +Nothing is enforced by tooling; match the file you are in. The conventions are those of Crafter.LintRules' full set, applied by hand. + +- The first two lines of every source are `// SPDX-License-Identifier: LGPL-3.0-only` and `// SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®`, with a space after `//` (as Crafter.Build spells it; Crafter.Math, Network and Asset do not), then a blank line. Every interface but `:Error` then carries a comment block saying what the partition owns, before `export module Crafter.DB:

;`; keep it true when the partition changes. +- Four-space indentation, no tabs, LF, a final newline, K&R braces, `if (x) return;` on one line when it fits, one declaration per statement, `if (` and `for (` with a space and no space before a call's `(`. +- `import std;` everywhere. The only C headers are in the global module fragment of `implementations/Crafter.DB-File.cpp` (after `module;`, with `#define _GNU_SOURCE 1`) and the two POSIX headers the forking tests include before their imports. `__builtin_offsetof` rather than ``. +- `PascalCase` for types, functions and concepts; `camelCase` for variables, members and constants alike (`inlineVarBytes`, `fnvOffset`, `rowFlagDeleted`), which is why the table statics are `version` and `orderedBy` while `Previous` and `Migrate` are a type and a function. Internal namespaces are `Reflect`, `Detail`, `Os` and `SchemaTests`. `enum class` only. Fixed-width integers; plain `int` and `long` appear only where POSIX or the language demand them (`main`, `waitpid`'s `status`, `operator++(int)`, what `sysconf` returns) and in the tests' `failures` counters. +- `std::format` and `std::println`; iostream appears only in the Benchmark's copy of the legacy format and in the tests' `Patch` helpers and fixture writers. Exception messages start with the path where one is known (`{}: …`; `Heap::Read`, `EncodeVar`, `CompactInto`, the reservation and the `msync` failure carry none). The `static_assert`s of `CheckTable`, `CopyByName` and `CheckEmplaceArgument` start with `Crafter.DB: ` and name the table type and the member (`Message` and `MemberMessage` in `:Schema`); a handful of internal limits assert without the prefix. +- Scalars and handles are read with `std::memcpy` into a local, and the flags byte through `std::atomic_ref` because readers race with `Erase`; the only reinterpreted pointers are the zero-copy views (`StringRef::View`, `BlobRef::View`, `DynamicRow::Text`) and the schema name pool. +- Hot paths carry no indirection: predicates are template parameters, column offsets are `constexpr`, and `HeapWriter::Grow` is the only virtual call, reached only when an extent fills. A change to the scan loop, `StringRef`'s comparisons, `Row::operator[]` or the encoders comes with evidence, the Benchmark before and after or the `-O3` assembly of the loop; the maintainer reviews performance claims at that level. +- Comments say why: the failure mode, the race, the rejected alternative (`MADV_DONTFORK` in `-File.cpp`, the `external` object in `:Reflect`, the sentinel iterator in `:Row`). The code carries long comments on purpose; do not strip them, write them for new non-obvious code, and give commit bodies the same treatment. + +## Design rules + +Decided deliberately; change them only with the maintainer's say-so. + +1. **Structs are the schema.** No macros, registration, traits to specialise or code generation. Every rule about a table struct's members is a `static_assert` in `CheckTable()` with a message naming the table and member, the size limits are `static_assert`s and consteval throws in `Layout` and `Arrange`, and a type with no `Formatter` is a clean concept failure. Misuse is a compile error, never a runtime surprise. +2. **One file per table, two regions, fixed addresses.** Rows and heap are chains of doubling extents appended at the end of the file and mapped `MAP_FIXED` into a `PROT_NONE` reservation, so row `i` is `base + i * rowSize` and no pointer moves while the table is open. No free-space management: `Compact()` right-sizes. Region slots 2 and 3 are reserved for a persisted index and a WAL. +3. **Fixed-width rows, 16-byte handles, a bump-arena heap.** Flags byte at offset 0, leaves stable-sorted by alignment, row size rounded to `rowAlign`. Strings and blobs are Umbra handles (length, 4-byte prefix, then 8 inline bytes or a heap offset), so values of up to 12 bytes live inline (strings, and blobs whose elements are aligned to 4 or less). The heap has no per-entry header; `Update`, `Set` and `Erase` orphan bytes that only `Compact()` reclaims. Little-endian only. +4. **The file format is a contract pinned twice**: by the `static_assert`s on every struct size and the load-bearing offsets in `:Format`, and by `tests/ShouldKeepLayoutGolden` with an independent spelling of the fingerprint and the exact handle bytes. `Kind` is append-only. The fingerprint covers names, kinds, offsets, sizes, flags, chrono period, blob element type, clock and the ordered column, and what it covers is what a `version` bump is required for. Changing any byte an existing file holds, or the fingerprint's inputs, is a format change: bump `formatVersion` (1 today, exact match required) and say so; never reinterpret old files silently. +5. **Durability by two commit records, not a journal.** An append is visible to in-process readers at once (release stores of the published counts) and durable at `Flush()`: `fdatasync`, the inactive commit slot with its CRC, `msync`. On open the newest valid commit wins and everything past it, uncommitted extents included, is discarded; an unclean close recounts tombstones. `Update` and `Set` commit their new heap bytes before the row points at them, and the row bytes themselves are not atomic against a power loss; that limitation is documented, and the WAL region is the v2 answer, not an `fsync` per row. +6. **A type-erased engine under a thin typed layer.** `TableFile` knows rows as `rowSize` bytes and columns through the file's own descriptors, so `Inspect`, `Verify`, `Compact`, `DynamicRow` and a migration from a struct that no longer exists work without the C++ type; `Table` adds compile-time offsets and encoders on top; `:File` knows nothing about tables at all. Keep each layer ignorant of the one above it. +7. **Queries are C++ and inline.** A predicate is a template parameter stored by value in the view: no `std::function`, no virtual call, no per-row call. Search keys are encoded once (`Key`, or automatically in a column expression). Bound extraction reads only comparisons on the `orderedBy` column in a top-level `&&` chain; `||` and `!` leave the whole predicate as a filter, which is always correct and only ever a narrowing. No SQL and no planner; hash indexes (`indexedBy`) are deferred, not refused. +8. **One writer, many in-process readers, a standard container's contract plus a little.** `flock` exclusive for a writer, shared for `readOnly`, between processes and between two `Table` objects in one. `Append`, `Emplace` and `Erase` are safe while other threads read (published counts, a monotonic tombstone flag, sentinel-terminated iteration that never compares against a moving end); `Update`, `Set`, `Compact` and `Clear` need exclusive access from the caller. `Row` views and `StringRef`s survive `Append` and `Erase`. +9. **Private by default, and never replace.** Files `0600` in `0700` directories, `O_NOFOLLOW`, `MADV_DONTDUMP`, creation through an `O_EXCL` temp file and a rename that fails when the target exists (racing creators share one file), rewrites through a temp file and `rename` (a visible table file is always valid). `MADV_DONTFORK` was tried and rejected: a forked child may read an inherited table, and its writes, `Flush` and destructor flush are refused by the owner-process check instead. +10. **Every byte from a file is checked before it is used**: header fields, both commit CRCs, extents against the committed length, every descriptor field in `SchemaInfo::Parse`, every heap handle against the published heap length (`Heap::Read`), the page-size granule against the running kernel. A new field read from a file gets its check and a case in `ShouldRejectSchemaChange` or `ShouldVerify`. +11. **Linux, no dependencies, no host split.** The library needs `flock`, `fallocate` and fixed-address `mmap` and says so; there is no `#ifdef` for another platform and no portability layer to grow. It depends on nothing, so a consumer's build fetches nothing for it. + +## Commit and PR guidelines + +- Branch off `master` (`claude/` is the pattern for agent work in the sibling repositories) and open the PR against `master` on the project's Forgejo. The repository is `Catcrafts/Crafter.DB` on the project's Forgejo: `forgejo@forgejo.catcrafts.net:Catcrafts/Crafter.DB.git` over SSH, `https://forgejo.catcrafts.net/Catcrafts/Crafter.DB.git` for consumers. A checkout whose `origin` still names the personal namespace the repository moved from can fetch (HTTPS redirects) but not push over SSH ("Push to create is not enabled"); repoint `origin`. Pushes go over SSH. +- The history has one commit, so take the voice from the sibling repositories: `type(scope): imperative summary` with `feat`, `fix`, `refactor`, `docs` or `test` and the partition as the scope (`storage`, `schema`, `query`, `migration`, `file`, `format`, `reflect`). Body: what was wrong, why, and why the fix has this shape, wrapped at about 75 columns, with the numbers when something was measured. Add a `Co-Authored-By:` trailer when an agent wrote it. +- Separate commits for the fix, its test and the README change when each stands on its own. The README is the manual: keep its module table, examples and "Performance" table true to the code. Update `AGENTS.consumer.md` in the same change when a signature, a default, a limit or an error behaviour consumers lean on changes; consumers pick it up on their next build. +- A format change (design rule 4) updates `ShouldKeepLayoutGolden` in the same commit, and the commit message says it is one. +- `crafter-build test` green before pushing. There is no CI. +- Never commit `bin/`, `build/` or `AGENTS.md`. + +## Security considerations + +- A table file is untrusted input: anything on disk may have been truncated, bit-flipped or crafted. Design rule 10 is the defence, and `ShouldRejectSchemaChange` and `ShouldVerify` are its regression suite. `Verify()` is the deep check and returns problems rather than throwing, so a tool can print them all. There is no per-row checksum: a flipped byte inside a scalar column goes unnoticed. +- The defaults keep data private (`0600`, `0700`, `O_NOFOLLOW` against a swapped symlink, `MADV_DONTDUMP` against core dumps). Do not loosen a default; a consumer that wants a readable file sets `filePermissions`. +- The lock is advisory (`flock`). A process that ignores it, or `cp`, can read the file mid-write, and what it reads is valid up to the writer's last `Flush` because only committed rows count on open. +- `reserveBytes` (32 GiB per region, two regions per table) is address space, not memory, but a tight `ulimit -v` or hundreds of open tables turns it into an `IOError` at open. +- `project.cpp` has no dependencies, so loading it executes nothing fetched from the network; keep it that way, or pin a `commit` on anything added. +- Tests write only under `fs::temp_directory_path()`, fork only to exercise locking and crashes, and never touch the network. + +## Gotchas + +- Adding a partition is three edits: the interface file, the `ifaces` array in `project.cpp` and the umbrella `Crafter.DB.cppm`; an implementation unit also goes into the `impls` array. Keep the order of the "Project overview" list: a partition imports only what is above it. +- `MemberName` depends on clang spelling `function_name()` as `… [P = &external.iban]` and `TypeName` on `… [T = ]`; the `static_assert`s at the bottom of `:Reflect` catch a change at the library's build. `Reflect::external` is `[[clang::no_destroy]] inline const T` on purpose: an undefined `extern` would be cheaper but warns at every instantiation. +- Leaf indices follow declaration order, depth first; column offsets do not. A `std::optional` contributes a presence leaf named `name?` before its payload, a nested aggregate flattens to `outer.inner`, a `std::array` of scalars to `name[i]`, and the flags byte is column 0 but no leaf. `Layout::Offset(leaf)` and `Column(leaf)` translate. +- Reordering two members of the same alignment class changes offsets and therefore the fingerprint; the README's `Transaction` has `currency` at offset 2, ahead of every string. Both follow from rule 3, and a `SchemaError` is the symptom in a consumer that forgot the `version` bump. +- `char` is always `Kind::Int8` whatever the platform's signedness, `wchar_t` depends on its size, `long double` is not storable, `std::optional>` is rejected (no distinct column names), and a `Formatter` may override `kind` only when its `Stored` is a single column. +- The `Transaction` inside `SchemaTests` and the `Probe` inside `:Reflect` are compile-time tests that run on every build of the library, not examples to copy from. +- `TableFile::Inspect` is a static member; the README's module table calls it `Inspect(path)`. The tests spell it `DB::TableFile::Inspect(path)`. +- Extents are aligned to the larger of 4 KiB and the running kernel's page size at creation, and `header.pageSize` records it. A file created on a 4 KiB kernel is refused on a 16 KiB one with an `IOError` (`not_supported`) rather than mapped badly; nothing tests that path. +- A region tops out at 32 extents (`IOError`, `file_too_large`, "Compact() it"); `Compact` sizes the first extent of the new file to the live payload, which resets the chain. A reservation that fills throws `IOError` (`not_enough_memory`) naming `OpenOptions::reserveBytes`. +- `Table::Open` is a loop for a reason: the first open fails with `no_such_file_or_directory`, the create may lose a race and fail with `file_exists`, and the second pass opens the other process's file. Both halves of the race are needed, and `TableFile::Create` never replaces. +- `RemoveStaleTemps` runs on every writable open, after the exclusive lock is taken, and deletes the `.tmp-*` siblings it can lock exclusively; the probe spares a temp another process is still writing. The twelve-hex-digit suffix comes from `Os::RandomBytes`. +- A migration's backup `.v.bak` is a hard link to the old inode taken before the rename, so it costs nothing until the original is replaced, and it fails on a filesystem without hard links. +- `Deleted()` reads the flags byte through a relaxed `atomic_ref`, `MarkDeleted` raises the monotonic `tombstones` flag before the byte, and `LiveRange::begin()` reads that flag when iteration starts rather than when the range was built, so a `Query` built before an `Erase` still skips the row. A scan that started before the `Erase` may yield it; that is documented, not a bug. +- `LowerBound`/`UpperBound` search every slot, tombstones included, because an erased row keeps its key; only `Compact` drops keys. A `Where` with bounds on the ordered column does the same. +- A `Query` over `Rows`, `Range`, `From` and the like can be iterated as a `const` object; one built with `Where` cannot, because `std::ranges::filter_view` has no const iteration (the deducing-`this` `requires` clauses in `Query` are what makes the first case work). `Query::Select` is rvalue-only: `std::move` a named query first. +- `Emplace` converts arguments implicitly, never with `static_cast`, so an `int` for an enum member or swapped arguments fail to compile with a message naming the member. `Append` of a range that is not both sized and random-access buffers it first, so the order check covers the whole batch before anything is written. +- `Set<&T::member>` copies only that member's leaves back into the slot (`CopyLeaves`) and leaves the flags byte alone, so an `Erase` that raced with it is not undone even though the contract gives `Set` exclusive access. Do not "simplify" it to a whole-row `memcpy`. +- After `fork()` the child's `Table` reads fine and refuses every write, `Flush` and its destructor's flush (`OwnedByThisProcess`): the parent's counters in the child are stale and must never reach the file. Do not make the destructor flush there. +- The Benchmark's `last timestamp (1000x open+read)` row is the one operation where the legacy format wins (about 2 ms against 30 ms): a mapped open costs one `mmap` per extent. The answer is to open once per process, which the README and the consumer notes say; do not try to win that row by changing the format. +- `crafter-build test` prints "Building std PCM" once per configuration (the library and each test) on every run; the step returns at once when the cached PCM is current, and only a toolchain update rebuilds it.