persistent browser filesystem
Some checks failed
CI / build-test-release (push) Failing after 17m50s

This commit is contained in:
Jorijn van der Graaf 2026-05-19 17:20:20 +02:00
commit 0e80877dca

View file

@ -6,28 +6,161 @@
const textEncoder = new TextEncoder();
// ─── Two VFS roots ─────────────────────────────────────────────────────
// fd=3 ⟶ read-only baked layer at "/". Populated from files.json at
// startup, keyed by basename — the bin-dir layout `cfg.assets`
// produces is flat enough that basename collapse doesn't
// collide for legitimate assets. Writes are rejected.
// fd=4 ⟶ read/write persistent layer at "/persistent". Backed by OPFS
// (Origin Private File System), keyed by relative path so
// nested directories work. The wasm sees an in-memory map
// (writes are synchronous from its POV); a background task
// mirrors dirty entries back to OPFS so they survive reload.
//
// wasi-libc walks preopens starting at fd=3 and resolves every open()
// against the longest matching preopen. "/persistent/foo.json" hits fd=4
// with relpath "foo.json"; "/Inter.ttf" hits fd=3 with relpath "Inter.ttf".
const readonlyVfs = new Map(); // basename → Uint8Array (immutable)
const persistentVfs = new Map(); // relpath → Uint8Array (mutable)
// Paths the wasm has mkdir'd. Our backing store is flat-keyed so dirs
// don't otherwise exist on their own — but libstdc++'s
// `create_directories` mkdir's, then re-stats each component, and
// expects the stat to report "directory". Tracking explicit mkdir'd
// paths here keeps that loop convergent.
const persistentDirs = new Set();
const ROOTS = {
3: { kind: "ro", map: readonlyVfs, name: "/" },
4: { kind: "rw", map: persistentVfs, name: "/persistent" },
};
function resolveKey(root, raw) {
if (root.kind === "ro") {
// Read-only baked layer: paths collapse to basename so the C++
// side can use "assets/foo.ctex" or "./foo.ctex" interchangeably.
return raw.split(/[\\/]/).filter(Boolean).pop() || raw;
}
// Persistent layer: keep the relative path so nested entries mirror
// OPFS's directory structure.
let key = raw.replace(/\\/g, "/");
while (key.startsWith("./")) key = key.slice(2);
while (key.startsWith("/")) key = key.slice(1);
return key;
}
// ─── OPFS mirror for the persistent layer ──────────────────────────────
let opfsRoot = null;
const dirtyPaths = new Set();
let flushTimer = null;
const FLUSH_DEBOUNCE_MS = 750;
async function opfsResolveDir(parts, create) {
if (!opfsRoot) return null;
let dir = opfsRoot;
for (const part of parts) {
if (!part) continue;
dir = await dir.getDirectoryHandle(part, { create });
}
return dir;
}
async function opfsHydrate() {
if (!navigator.storage || !navigator.storage.getDirectory) {
console.warn("[wasi] OPFS unavailable — persistent writes will not survive reload");
return;
}
try {
opfsRoot = await navigator.storage.getDirectory();
} catch (e) {
console.warn("[wasi] OPFS getDirectory failed:", e?.message);
return;
}
async function walk(dir, prefix) {
for await (const [name, handle] of dir.entries()) {
const full = prefix ? `${prefix}/${name}` : name;
if (handle.kind === "file") {
try {
const file = await handle.getFile();
persistentVfs.set(full, new Uint8Array(await file.arrayBuffer()));
} catch (e) {
console.warn(`[wasi] OPFS read failed for ${full}:`, e?.message);
}
} else if (handle.kind === "directory") {
await walk(handle, full);
}
}
}
await walk(opfsRoot, "");
}
async function opfsFlushOne(path) {
if (!opfsRoot) return;
const parts = path.split("/").filter(Boolean);
if (parts.length === 0) return;
const data = persistentVfs.get(path);
if (data === undefined) {
// Removed entry — delete from OPFS, walking parents.
try {
const parent = await opfsResolveDir(parts.slice(0, -1), false);
await parent.removeEntry(parts[parts.length - 1]);
} catch (e) {
// Missing entry is fine; other errors get reported.
if (e && e.name !== "NotFoundError") {
console.warn(`[wasi] OPFS remove failed for ${path}:`, e?.message);
}
}
return;
}
try {
const dir = await opfsResolveDir(parts.slice(0, -1), true);
const fh = await dir.getFileHandle(parts[parts.length - 1], { create: true });
const w = await fh.createWritable();
await w.write(data);
await w.close();
} catch (e) {
console.warn(`[wasi] OPFS write failed for ${path}:`, e?.message);
}
}
async function opfsFlushAll() {
if (!opfsRoot) return;
const batch = Array.from(dirtyPaths);
dirtyPaths.clear();
for (const path of batch) {
await opfsFlushOne(path);
}
}
function scheduleFlush() {
if (flushTimer !== null) return;
flushTimer = setTimeout(() => {
flushTimer = null;
opfsFlushAll();
}, FLUSH_DEBOUNCE_MS);
}
function markDirty(path) {
dirtyPaths.add(path);
scheduleFlush();
}
class Wasi {
#encodedStdin;
#envEncodedStrings;
#argEncodedStrings;
instance;
// VFS: file basename → Uint8Array, populated by EnableWasiBrowserRuntime's
// preload step before instantiateStreaming runs. Used by path_open / fd_read
// / fd_seek / fd_tell / fd_filestat_get / fd_close so wasi-libc's
// std::ifstream and friends actually work in the browser.
vfs;
#fdTable;
#nextFd;
#decoder;
constructor({ env, stdin, args, vfs }) {
constructor({ env, stdin, args }) {
this.#encodedStdin = textEncoder.encode(stdin);
const envStrings = Object.entries(env).map(([k, v]) => `${k}=${v}`);
this.#envEncodedStrings = envStrings.map(s => textEncoder.encode(s + "\0"));
this.#argEncodedStrings = args.map(s => textEncoder.encode(s + "\0"));
this.vfs = vfs || new Map();
this.#fdTable = new Map();
this.#nextFd = 100; // stay clear of stdio
this.#nextFd = 100; // stay clear of stdio + preopens
this.#decoder = new TextDecoder();
this.bind();
}
@ -40,10 +173,12 @@ class Wasi {
const m = [
"args_get", "args_sizes_get",
"environ_get", "environ_sizes_get",
"fd_read", "fd_write", "fd_close", "fd_seek", "fd_tell",
"fd_read", "fd_write", "fd_pwrite",
"fd_close", "fd_seek", "fd_tell",
"fd_filestat_get", "fd_fdstat_get",
"fd_prestat_get", "fd_prestat_dir_name",
"path_open",
"path_open", "path_create_directory", "path_remove_directory",
"path_unlink_file", "path_rename", "path_filestat_get",
];
for (const name of m) this[name] = this[name].bind(this);
}
@ -70,6 +205,7 @@ class Wasi {
return 0;
}
fd_write(fd, iovsPtr, iovsLength, bytesWrittenPtr) {
const dv = new DataView(this.instance.exports.memory.buffer);
const iovs = new Uint32Array(this.instance.exports.memory.buffer, iovsPtr, iovsLength * 2);
if (fd === 1 || fd === 2) {
let text = "";
@ -81,12 +217,63 @@ class Wasi {
text += decoder.decode(new Int8Array(this.instance.exports.memory.buffer, offset, length));
totalBytesWritten += length;
}
const dataView = new DataView(this.instance.exports.memory.buffer);
dataView.setInt32(bytesWrittenPtr, totalBytesWritten, true);
dv.setInt32(bytesWrittenPtr, totalBytesWritten, true);
(fd === 2 ? console.error : console.log)(text);
}
return 0;
}
const entry = this.#fdTable.get(fd);
if (!entry) {
dv.setInt32(bytesWrittenPtr, 0, true);
return 8; // EBADF
}
const root = ROOTS[entry.rootFd];
if (!root || root.kind !== "rw") {
dv.setInt32(bytesWrittenPtr, 0, true);
return 76; // EROFS — close enough; libc surfaces this as "no permission"
}
// Sum iov lengths so we can grow the entry once.
let total = 0;
for (let i = 0; i < iovsLength * 2; i += 2) total += iovs[i + 1];
let data = root.map.get(entry.name) ?? new Uint8Array(0);
const end = entry.offset + total;
if (data.byteLength < end) {
const grown = new Uint8Array(end);
grown.set(data, 0);
data = grown;
root.map.set(entry.name, data);
}
const memory = new Uint8Array(this.instance.exports.memory.buffer);
let written = 0;
for (let i = 0; i < iovsLength * 2; i += 2) {
const offset = iovs[i];
const length = iovs[i + 1];
data.set(memory.subarray(offset, offset + length), entry.offset + written);
written += length;
}
entry.offset += written;
dv.setInt32(bytesWrittenPtr, written, true);
markDirty(entry.name);
return 0;
}
fd_pwrite(fd, iovsPtr, iovsLength, offsetArg, bytesWrittenPtr) {
// pwrite is positioned write — same as fd_write but the seek pos
// isn't advanced. We just temporarily override entry.offset.
const entry = this.#fdTable.get(fd);
if (!entry) {
new DataView(this.instance.exports.memory.buffer)
.setInt32(bytesWrittenPtr, 0, true);
return 8;
}
const saved = entry.offset;
entry.offset = typeof offsetArg === "bigint" ? Number(offsetArg) : Number(offsetArg);
const rc = this.fd_write(fd, iovsPtr, iovsLength, bytesWrittenPtr);
entry.offset = saved;
return rc;
}
fd_read(fd, iovsPtr, iovsLength, bytesReadPtr) {
const memory = new Uint8Array(this.instance.exports.memory.buffer);
const iovs = new Uint32Array(this.instance.exports.memory.buffer, iovsPtr, iovsLength * 2);
@ -110,7 +297,12 @@ class Wasi {
dataView.setInt32(bytesReadPtr, 0, true);
return 8; // EBADF
}
const file = this.vfs.get(entry.name);
const root = ROOTS[entry.rootFd];
const file = root ? root.map.get(entry.name) : undefined;
if (!file) {
dataView.setInt32(bytesReadPtr, 0, true);
return 0; // EOF
}
for (let i = 0; i < iovsLength * 2; i += 2) {
const offset = iovs[i];
const length = iovs[i + 1];
@ -133,8 +325,10 @@ class Wasi {
fd_fdstat_get(fd, statPtr) {
// 24 bytes: filetype(1) + flags(2) + padding + rights_base(8) + rights_inheriting(8).
const dv = new DataView(this.instance.exports.memory.buffer);
const isFile = this.#fdTable.has(fd);
const filetype = isFile ? 4 : 0; // 4 = regular_file
let filetype;
if (ROOTS[fd]) filetype = 3; // directory — preopen mount
else if (this.#fdTable.has(fd)) filetype = 4; // regular_file
else filetype = 0;
dv.setUint8(statPtr + 0, filetype);
dv.setUint16(statPtr + 2, 0, true);
dv.setBigUint64(statPtr + 8, 0xFFFFFFFFFFFFFFFFn, true);
@ -143,22 +337,22 @@ class Wasi {
}
// wasi-libc walks preopens starting at fd=3 until fd_prestat_get returns
// EBADF, then resolves every relative open against one of the discovered
// dirs. Without at least one preopen, std::ifstream et al can't open any
// path. We expose a single "/" preopen on fd=3, rooted at the VFS map.
// dirs. Two preopens: fd=3 ("/") for baked assets, fd=4 ("/persistent")
// for OPFS-backed mutable state.
fd_prestat_get(fd, prestatPtr) {
const dv = new DataView(this.instance.exports.memory.buffer);
if (fd === 3) {
const root = ROOTS[fd];
if (!root) return 8; // EBADF — terminates the walk
// prestat_t: tag(u8) + padding to 4 + u.dir.pr_name_len(u32). 8 bytes.
dv.setUint8(prestatPtr + 0, 0); // tag = preopentype_dir
dv.setUint32(prestatPtr + 4, 1, true); // strlen("/")
dv.setUint32(prestatPtr + 4, root.name.length, true);
return 0;
}
return 8; // EBADF
}
fd_prestat_dir_name(fd, pathPtr, pathLen) {
if (fd !== 3) return 8;
const root = ROOTS[fd];
if (!root) return 8;
const memory = new Uint8Array(this.instance.exports.memory.buffer);
const name = textEncoder.encode("/");
const name = textEncoder.encode(root.name);
memory.set(name.subarray(0, Math.min(name.byteLength, pathLen)), pathPtr);
return 0;
}
@ -175,16 +369,22 @@ class Wasi {
return 8;
}
const delta = typeof offsetLow === "bigint" ? Number(offsetLow) : Number(offsetLow);
const file = this.vfs.get(entry.name);
const root = ROOTS[entry.rootFd];
const file = root ? root.map.get(entry.name) : undefined;
const size = file ? file.byteLength : 0;
let newOff;
switch (whence) {
case 0: newOff = delta; break; // SEEK_SET
case 1: newOff = entry.offset + delta; break; // SEEK_CUR
case 2: newOff = file.byteLength + delta; break; // SEEK_END
case 2: newOff = size + delta; break; // SEEK_END
default: return 28; // EINVAL
}
if (newOff < 0) newOff = 0;
if (newOff > file.byteLength) newOff = file.byteLength;
// Writeable fds may seek past EOF (the next write extends the
// backing buffer); read-only fds clamp to EOF.
if (!root || root.kind !== "rw") {
if (newOff > size) newOff = size;
}
entry.offset = newOff;
dv.setBigUint64(newOffsetPtr, BigInt(newOff), true);
return 0;
@ -203,14 +403,22 @@ class Wasi {
fd_filestat_get(fd, statPtr) {
// Layout: dev(8) ino(8) filetype(1) +pad nlink(8) size(8) atim(8) mtim(8) ctim(8) = 64 bytes.
const dv = new DataView(this.instance.exports.memory.buffer);
let filetype = 4; // regular_file (default)
let size = 0;
if (ROOTS[fd]) {
filetype = 3; // directory — preopen mount itself
} else {
const entry = this.#fdTable.get(fd);
if (!entry) return 8;
const file = this.vfs.get(entry.name);
const root = ROOTS[entry.rootFd];
const file = root ? root.map.get(entry.name) : undefined;
size = file ? file.byteLength : 0;
}
dv.setBigUint64(statPtr + 0, 0n, true);
dv.setBigUint64(statPtr + 8, 0n, true);
dv.setUint8(statPtr + 16, 4); // filetype = regular_file
dv.setUint8(statPtr + 16, filetype);
dv.setBigUint64(statPtr + 24, 1n, true);
dv.setBigUint64(statPtr + 32, BigInt(file.byteLength), true);
dv.setBigUint64(statPtr + 32, BigInt(size), true);
dv.setBigUint64(statPtr + 40, 0n, true);
dv.setBigUint64(statPtr + 48, 0n, true);
dv.setBigUint64(statPtr + 56, 0n, true);
@ -219,36 +427,151 @@ class Wasi {
fd_filestat_set_size() { return 0; }
fd_filestat_set_times() { return 0; }
fd_pread() { return 0; }
fd_pwrite() { return 0; }
fd_readdir() { return 0; }
fd_renumber() { return 0; }
fd_sync() { return 0; }
fd_tell() { return 0; }
path_create_directory() { return 0; }
path_filestat_get() { return 0; }
path_filestat_set_times() { return 0; }
path_link() { return 0; }
path_open(_dirfd, _dirflags, pathPtr, pathLen, _oflags, _rightsBase, _rightsInh, _fdflags, openedFdPtr) {
path_filestat_get(dirfd, _dirflags, pathPtr, pathLen, statPtr) {
const root = ROOTS[dirfd];
if (!root) return 8;
const memory = new Uint8Array(this.instance.exports.memory.buffer);
const raw = this.#decoder.decode(memory.subarray(pathPtr, pathPtr + pathLen));
const key = resolveKey(root, raw);
const file = root.map.get(key);
const dv = new DataView(this.instance.exports.memory.buffer);
// Resolve filetype: 4 = regular_file, 3 = directory. The
// preopen mount itself (empty key) is always a directory. Any
// key that is a prefix of another key acts as a directory too
// — `std::filesystem::create_directories` walks parent
// components and stats each one, so reporting them as dirs
// keeps libc happy even though our backing store is flat-keyed.
let filetype;
let size = 0n;
if (file !== undefined) {
filetype = 4;
size = BigInt(file.byteLength);
} else if (key === "" || persistentDirs.has(key)) {
filetype = 3;
} else {
const prefix = key + "/";
let isDir = false;
for (const k of root.map.keys()) {
if (k.startsWith(prefix)) { isDir = true; break; }
}
if (!isDir) return 44; // ENOENT
filetype = 3;
}
dv.setBigUint64(statPtr + 0, 0n, true);
dv.setBigUint64(statPtr + 8, 0n, true);
dv.setUint8(statPtr + 16, filetype);
dv.setBigUint64(statPtr + 24, 1n, true);
dv.setBigUint64(statPtr + 32, size, true);
dv.setBigUint64(statPtr + 40, 0n, true);
dv.setBigUint64(statPtr + 48, 0n, true);
dv.setBigUint64(statPtr + 56, 0n, true);
return 0;
}
path_create_directory(dirfd, pathPtr, pathLen) {
const root = ROOTS[dirfd];
if (!root) return 8;
if (root.kind !== "rw") return 76;
// Flat-keyed Map: directories don't need to be "created" for
// files to exist under them. Track the path in persistentDirs
// so a subsequent path_filestat_get reports it as a directory
// — that's what libstdc++'s create_directories looks for to
// confirm the mkdir landed.
const memory = new Uint8Array(this.instance.exports.memory.buffer);
const raw = this.#decoder.decode(memory.subarray(pathPtr, pathPtr + pathLen));
const key = resolveKey(root, raw);
if (key) persistentDirs.add(key);
return 0;
}
path_remove_directory(dirfd, pathPtr, pathLen) {
const root = ROOTS[dirfd];
if (!root || root.kind !== "rw") return 8;
const memory = new Uint8Array(this.instance.exports.memory.buffer);
const raw = this.#decoder.decode(memory.subarray(pathPtr, pathPtr + pathLen));
const key = resolveKey(root, raw);
// Delete every entry under this prefix and mark them for OPFS removal.
const prefix = key.endsWith("/") ? key : key + "/";
for (const k of [...root.map.keys()]) {
if (k === key || k.startsWith(prefix)) {
root.map.delete(k);
markDirty(k);
}
}
return 0;
}
path_unlink_file(dirfd, pathPtr, pathLen) {
const root = ROOTS[dirfd];
if (!root || root.kind !== "rw") return 8;
const memory = new Uint8Array(this.instance.exports.memory.buffer);
const raw = this.#decoder.decode(memory.subarray(pathPtr, pathPtr + pathLen));
const key = resolveKey(root, raw);
if (!root.map.has(key)) return 44; // ENOENT
root.map.delete(key);
markDirty(key);
return 0;
}
path_rename(oldDirfd, oldPathPtr, oldPathLen,
newDirfd, newPathPtr, newPathLen) {
const oldRoot = ROOTS[oldDirfd];
const newRoot = ROOTS[newDirfd];
if (!oldRoot || !newRoot) return 8;
if (oldRoot.kind !== "rw" || newRoot.kind !== "rw") return 76;
const memory = new Uint8Array(this.instance.exports.memory.buffer);
const oldRaw = this.#decoder.decode(memory.subarray(oldPathPtr, oldPathPtr + oldPathLen));
const newRaw = this.#decoder.decode(memory.subarray(newPathPtr, newPathPtr + newPathLen));
const oldKey = resolveKey(oldRoot, oldRaw);
const newKey = resolveKey(newRoot, newRaw);
const data = oldRoot.map.get(oldKey);
if (data === undefined) return 44;
oldRoot.map.delete(oldKey);
newRoot.map.set(newKey, data);
markDirty(oldKey);
markDirty(newKey);
return 0;
}
path_open(dirfd, _dirflags, pathPtr, pathLen, oflags, _rightsBase, _rightsInh, _fdflags, openedFdPtr) {
const memory = new Uint8Array(this.instance.exports.memory.buffer);
const dv = new DataView(this.instance.exports.memory.buffer);
const root = ROOTS[dirfd];
if (!root) {
dv.setInt32(openedFdPtr, -1, true);
return 8; // EBADF
}
const raw = this.#decoder.decode(memory.subarray(pathPtr, pathPtr + pathLen));
// wasi-libc may pass paths like "./font.ttf" or "/font.ttf" or "font.ttf"
// depending on how std::filesystem::path was constructed. Reduce to basename.
const base = raw.split(/[\\/]/).filter(Boolean).pop() || raw;
if (!this.vfs.has(base)) {
const key = resolveKey(root, raw);
const O_CREAT = 0x1;
const O_TRUNC = 0x8;
const create = (oflags & O_CREAT) !== 0;
const trunc = (oflags & O_TRUNC) !== 0;
let present = root.map.has(key);
if (!present) {
if (!create || root.kind !== "rw") {
dv.setInt32(openedFdPtr, -1, true);
return 44; // ENOENT
}
root.map.set(key, new Uint8Array(0));
markDirty(key);
present = true;
} else if (trunc && root.kind === "rw") {
root.map.set(key, new Uint8Array(0));
markDirty(key);
}
const fd = this.#nextFd++;
this.#fdTable.set(fd, { name: base, offset: 0 });
this.#fdTable.set(fd, { rootFd: dirfd, name: key, offset: 0 });
dv.setInt32(openedFdPtr, fd, true);
return 0;
}
path_readlink() { return 0; }
path_remove_directory() { return 0; }
path_rename() { return 0; }
path_symlink() { return 0; }
path_unlink_file() { return 0; }
path_link() { return 0; }
path_filestat_set_times() { return 0; }
poll_oneoff() { return 0; }
sched_yield() { return 0; }
random_get() { return 0; }
@ -286,6 +609,9 @@ class Wasi {
// Crafter::Window::StartSync on DOM, which hands the loop to rAF
// and then exits main without running static destructors).
console.log(`[wasi] proc_exit(${code})`);
// Trigger a final flush so anything written this session has the
// best chance of reaching OPFS before the page is torn down.
opfsFlushAll();
const e = new Error(`wasi proc_exit(${code})`);
e.crafterWasiExit = code;
throw e;
@ -298,16 +624,16 @@ if (!wasmUrl) {
}
// Preload asset files listed in files.json (emitted by
// EnableWasiBrowserRuntime) into an in-memory VFS so wasi-libc's file
// syscalls work against them. Browser builds otherwise can't open assets
// shipped alongside the .wasm — sync XHR is too deprecated to rely on.
// EnableWasiBrowserRuntime) into the read-only baked layer so wasi-libc's
// file syscalls work against them. Browser builds otherwise can't open
// assets shipped alongside the .wasm — sync XHR is too deprecated to rely
// on.
//
// Manifest entries are relative paths under the bin dir (the layout
// `cfg.assets` produces — e.g. "assets/Inter.ttf",
// "mods/3DForts_Base/foo.cmesh"). We fetch each at its full path but
// key the VFS by basename so `path_open`'s basename-reduction can find
// it regardless of the C++ side's cwd or prefix.
const vfs = new Map();
// "mods/3DForts_Base/foo.cmesh"). We fetch each at its full path but key
// the read-only map by basename so `path_open`'s basename-reduction can
// find it regardless of the C++ side's cwd or prefix.
try {
const manifestResp = await fetch("files.json");
if (manifestResp.ok) {
@ -319,14 +645,25 @@ try {
return;
}
const base = name.split(/[\\/]/).filter(Boolean).pop() || name;
vfs.set(base, new Uint8Array(await r.arrayBuffer()));
readonlyVfs.set(base, new Uint8Array(await r.arrayBuffer()));
}));
}
} catch (e) {
console.warn("[wasi] no files.json manifest (or fetch failed); file I/O syscalls will return ENOENT:", e.message);
}
const wasi = new Wasi({ stdin: "", env: {}, args: [], vfs });
// Hydrate the persistent layer from OPFS. Must complete before main()
// runs so any std::ifstream on /persistent/* sees the on-disk bytes.
await opfsHydrate();
// Best-effort final flush as the page is torn down. `pagehide` is the
// modern equivalent of `unload`; it fires on tab close + bfcache eviction
// + cross-document navigation. The browser MAY kill the page before the
// async writes complete — the periodic FLUSH_DEBOUNCE_MS timer is the
// real durability guarantee for writes that happen close to shutdown.
window.addEventListener("pagehide", () => { opfsFlushAll(); });
const wasi = new Wasi({ stdin: "", env: {}, args: [] });
// Modules that need env imports (Crafter.Graphics DOM mode, etc.) ship a
// co-located env.js that sets `window.crafter_webbuild_env`. The