From 2b7e37e3b96cab7394b3cb14e36920f1a8903ad1 Mon Sep 17 00:00:00 2001 From: Jorijn van der Graaf Date: Tue, 26 May 2026 22:50:08 +0200 Subject: [PATCH] wasm fixes --- implementations/Crafter.Build-Clang.cpp | 102 +++++++++- wasi-runtime/index.html.in | 4 +- wasi-runtime/runtime.js | 251 ++++++++++++++++++++---- 3 files changed, 311 insertions(+), 46 deletions(-) diff --git a/implementations/Crafter.Build-Clang.cpp b/implementations/Crafter.Build-Clang.cpp index 8ac6c40..db81f13 100644 --- a/implementations/Crafter.Build-Clang.cpp +++ b/implementations/Crafter.Build-Clang.cpp @@ -1125,9 +1125,20 @@ void Crafter::EnableWasiBrowserRuntime(Configuration& cfg) { }; walk(&cfg); + // Per-build cache-busting token. Stamped onto every script src + the + // wasm URL so a regular browser reload sees fresh files even though + // the dev server (python -m http.server) sends no Cache-Control + // headers. Using ms-since-epoch is enough to be unique per build + // without invoking any version-control machinery. + const std::string buildId = std::to_string( + std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count()); + std::string envScriptTags; for (const std::string& name : envScripts) { - envScriptTags += std::format(" \n", name); + envScriptTags += std::format( + " \n", + name, buildId); } // Walk the dep graph again for non-JS assets — these get pre-loaded by @@ -1215,6 +1226,7 @@ void Crafter::EnableWasiBrowserRuntime(Configuration& cfg) { std::string html = buf.str(); html = std::regex_replace(html, std::regex(R"(\{\{WASM\}\})"), cfg.outputName + ".wasm"); html = std::regex_replace(html, std::regex(R"(\{\{ENV_SCRIPTS\}\})"), envScriptTags); + html = std::regex_replace(html, std::regex(R"(\{\{BUILDID\}\})"), buildId); std::ofstream out(htmlPath); out << html; out.close(); @@ -1467,20 +1479,81 @@ int Crafter::Run(int argc, char** argv) { return basePort; }; const int port = findFreePort(8080, 16); + // Cross-origin isolation: the browser coarsens + // performance.now() (and chrono::steady_clock under wasi) + // to ~0.1ms unless the response carries COOP/COEP, which + // floors the --timing overlay's sub-ms phase counters to + // zero. Emitting same-origin / require-corp drops the + // resolution to ~5µs and also unlocks SharedArrayBuffer + // for any future threading work. CORP keeps the local + // asset fetches passing under require-corp. + auto writeFile = [](const fs::path& p, std::string_view contents) { + std::ofstream f(p, std::ios::binary | std::ios::trunc); + f.write(contents.data(), static_cast(contents.size())); + }; std::string cmd; std::string_view picked; + bool isolated = false; if (have("caddy")) { picked = "caddy"; - cmd = std::format("caddy file-server --listen :{} --root {}", port, absDir.string()); - } else if (have("python3")) { - picked = "python3"; - cmd = std::format("python3 -m http.server --directory {} {}", absDir.string(), port); - } else if (have("python")) { - picked = "python"; - cmd = std::format("python -m http.server --directory {} {}", absDir.string(), port); + isolated = true; + // caddy file-server has no --header flag; write a + // Caddyfile next to the build output. Adapter is + // inferred from the .caddyfile extension when run + // via `caddy run`. + fs::path cf = absDir / "Caddyfile.coi"; + writeFile(cf, std::format( + ":{} {{\n" + " root * {}\n" + " header Cross-Origin-Opener-Policy \"same-origin\"\n" + " header Cross-Origin-Embedder-Policy \"require-corp\"\n" + " header Cross-Origin-Resource-Policy \"same-origin\"\n" + " header Cache-Control \"no-store\"\n" + " file_server\n" + "}}\n", + port, absDir.string())); + cmd = std::format("caddy run --config {} --adapter caddyfile", + cf.string()); + } else if (have("python3") || have("python")) { + std::string_view py = have("python3") ? "python3" : "python"; + picked = py; + isolated = true; + // Inline a tiny SimpleHTTPRequestHandler subclass + // that appends the COI headers on every response. + // Lives in absDir so the user can re-run it manually + // (`python3 absDir/.serve-coi.py 8080`). + fs::path sp = absDir / ".serve-coi.py"; + writeFile(sp, + "import http.server, socketserver, sys, os\n" + "class H(http.server.SimpleHTTPRequestHandler):\n" + " def end_headers(self):\n" + " self.send_header('Cross-Origin-Opener-Policy', 'same-origin')\n" + " self.send_header('Cross-Origin-Embedder-Policy', 'require-corp')\n" + " self.send_header('Cross-Origin-Resource-Policy', 'same-origin')\n" + " self.send_header('Cache-Control', 'no-store')\n" + " super().end_headers()\n" + "socketserver.TCPServer.allow_reuse_address = True\n" + "port = int(sys.argv[1]) if len(sys.argv) > 1 else 8080\n" + "os.chdir(os.path.dirname(os.path.abspath(__file__)))\n" + "with socketserver.TCPServer(('', port), H) as s:\n" + " s.serve_forever()\n"); + cmd = std::format("{} {} {}", py, sp.string(), port); } else if (have("php")) { picked = "php"; - cmd = std::format("php -S 0.0.0.0:{} -t {}", port, absDir.string()); + // php -S supports a router script — emit one that + // sets COI headers then delegates to the built-in + // static-file handler by returning false. + fs::path rp = absDir / ".serve-coi.php"; + writeFile(rp, + " {{WASM}} - -{{ENV_SCRIPTS}} + +{{ENV_SCRIPTS}} diff --git a/wasi-runtime/runtime.js b/wasi-runtime/runtime.js index 4ffbb47..fd8445c 100644 --- a/wasi-runtime/runtime.js +++ b/wasi-runtime/runtime.js @@ -35,20 +35,48 @@ const ROOTS = { 4: { kind: "rw", map: persistentVfs, name: "/persistent" }, }; -function resolveKey(root, raw) { - if (root.kind === "ro") { - // Read-only baked layer: paths collapse to basename so the C++ - // side can use "assets/foo.ctex" or "./foo.ctex" interchangeably. - return raw.split(/[\\/]/).filter(Boolean).pop() || raw; - } - // Persistent layer: keep the relative path so nested entries mirror - // OPFS's directory structure. +function resolveKey(_root, raw) { + // Both VFS layers key by relative path now. We did keep the read- + // only side basename-collapsed up to Phase 1, but that broke as + // soon as two bundled dirs shipped a file with the same basename + // (e.g. maps/A/map.json vs maps/B/map.json — only one survives the + // preload). Full-path keying matches what files.json actually + // serves and what cfg.assets actually deploys. let key = raw.replace(/\\/g, "/"); while (key.startsWith("./")) key = key.slice(2); while (key.startsWith("/")) key = key.slice(1); return key; } +// Directory tree for the read-only baked layer. Built after files.json +// preload completes — each path like "maps/A/map.json" registers as: +// "" → { "maps": "dir" } +// "maps" → { "A": "dir" } +// "maps/A" → { "map.json":"file" } +// Used by path_open (recognize directory paths) and fd_readdir +// (iterate entries). +const readonlyDirs = new Map(); +function roBuildDirTree() { + readonlyDirs.clear(); + const addChild = (parent, name, kind) => { + let entries = readonlyDirs.get(parent); + if (!entries) { + entries = new Map(); + readonlyDirs.set(parent, entries); + } + if (!entries.has(name)) entries.set(name, kind); + }; + for (const path of readonlyVfs.keys()) { + const parts = path.split("/").filter(Boolean); + for (let i = 0; i < parts.length; ++i) { + const parent = parts.slice(0, i).join("/"); + const name = parts[i]; + const kind = (i === parts.length - 1) ? "file" : "dir"; + addChild(parent, name, kind); + } + } +} + // ─── OPFS mirror for the persistent layer ────────────────────────────── let opfsRoot = null; const dirtyPaths = new Set(); @@ -173,12 +201,15 @@ class Wasi { const m = [ "args_get", "args_sizes_get", "environ_get", "environ_sizes_get", - "fd_read", "fd_write", "fd_pwrite", + "fd_read", "fd_write", "fd_pwrite", "fd_readdir", "fd_close", "fd_seek", "fd_tell", "fd_filestat_get", "fd_fdstat_get", "fd_prestat_get", "fd_prestat_dir_name", "path_open", "path_create_directory", "path_remove_directory", "path_unlink_file", "path_rename", "path_filestat_get", + // clock_res_get + clock_time_get touch this.instance.exports + // .memory; the old stubs were no-ops and didn't need binding. + "clock_res_get", "clock_time_get", ]; for (const name of m) this[name] = this[name].bind(this); } @@ -326,9 +357,13 @@ class Wasi { // 24 bytes: filetype(1) + flags(2) + padding + rights_base(8) + rights_inheriting(8). const dv = new DataView(this.instance.exports.memory.buffer); let filetype; - if (ROOTS[fd]) filetype = 3; // directory — preopen mount - else if (this.#fdTable.has(fd)) filetype = 4; // regular_file - else filetype = 0; + if (ROOTS[fd]) { + filetype = 3; // directory — preopen mount + } else { + const entry = this.#fdTable.get(fd); + if (entry) filetype = entry.isDir ? 3 : 4; + else filetype = 0; + } dv.setUint8(statPtr + 0, filetype); dv.setUint16(statPtr + 2, 0, true); dv.setBigUint64(statPtr + 8, 0xFFFFFFFFFFFFFFFFn, true); @@ -356,8 +391,42 @@ class Wasi { memory.set(name.subarray(0, Math.min(name.byteLength, pathLen)), pathPtr); return 0; } - clock_res_get() { return 0; } - clock_time_get() { return 0; } + // WASI clock_res_get(clock_id, resPtr): writes a u64 nanosecond + // resolution to *resPtr. We back every clock with performance.now() + // which the platform documents as ≥5µs resolution on modern browsers + // — round-trip via Math.round() loses sub-µs precision but anything + // more aggressive would mis-represent the actual host resolution. + clock_res_get(_clockId, resPtr) { + const dv = new DataView(this.instance.exports.memory.buffer); + dv.setBigUint64(resPtr, 1000n, /*littleEndian*/ true); // 1µs in ns + return 0; + } + // WASI clock_time_get(clock_id, precision, timePtr): writes a u64 + // nanosecond timestamp. The stub used to return 0 unconditionally, + // which made every std::chrono::high_resolution_clock::now() in wasm + // return the same value — Physics::Update read delta=0 each frame + // and never advanced. Use performance.now() for MONOTONIC and a + // Date.now()-relative reference for REALTIME so std::chrono actually + // works. + clock_time_get(clockId, _precision, timePtr) { + const dv = new DataView(this.instance.exports.memory.buffer); + // clock IDs per wasi-snapshot-preview1: + // 0 = realtime, 1 = monotonic, 2 = process_cputime_id, 3 = thread_cputime_id + let nanos; + if (clockId === 0) { + // Date.now() returns ms since the Unix epoch. + nanos = BigInt(Date.now()) * 1000000n; + } else { + // performance.now() returns ms since the page navigation + // start with sub-ms precision. Anchor against + // performance.timeOrigin so caller sees a clean monotonic + // walltime equivalent. + const ms = performance.now(); + nanos = BigInt(Math.round(ms * 1e6)); + } + dv.setBigUint64(timePtr, nanos, /*littleEndian*/ true); + return 0; + } fd_seek(fd, offsetLow, whence, newOffsetPtr) { // offsetLow is a BigInt under wasi-snapshot-preview1's two-i32 ABI? // No — wasi-snapshot-preview1 fd_seek's signature is (fd, filedelta:i64, @@ -410,9 +479,13 @@ class Wasi { } else { const entry = this.#fdTable.get(fd); if (!entry) return 8; - const root = ROOTS[entry.rootFd]; - const file = root ? root.map.get(entry.name) : undefined; - size = file ? file.byteLength : 0; + if (entry.isDir) { + filetype = 3; + } else { + const root = ROOTS[entry.rootFd]; + const file = root ? root.map.get(entry.name) : undefined; + size = file ? file.byteLength : 0; + } } dv.setBigUint64(statPtr + 0, 0n, true); dv.setBigUint64(statPtr + 8, 0n, true); @@ -427,7 +500,85 @@ class Wasi { fd_filestat_set_size() { return 0; } fd_filestat_set_times() { return 0; } fd_pread() { return 0; } - fd_readdir() { return 0; } + fd_readdir(fd, bufPtr, bufLen, cookieArg, bytesUsedPtr) { + // wasi dirent: dircookie(8) + inode(8) + namlen(4) + filetype(1) + // = 24 bytes header (the 3-byte tail is implicit padding before + // the variable-length name). The cookie returned in each entry + // is the value the caller passes in to read the *next* entry. + // We return at most bufLen bytes; libc reads, processes, calls + // back with the last entry's cookie, repeats until short read. + const dv = new DataView(this.instance.exports.memory.buffer); + const mem = new Uint8Array(this.instance.exports.memory.buffer); + const entry = this.#fdTable.get(fd); + if (!entry || !entry.isDir) { + dv.setInt32(bytesUsedPtr, 0, true); + return 8; // EBADF + } + const root = ROOTS[entry.rootFd]; + if (!root) { + dv.setInt32(bytesUsedPtr, 0, true); + return 8; + } + + // Gather the immediate children of `entry.name`. + let children; + if (root.kind === "ro") { + const dirMap = readonlyDirs.get(entry.name); + children = dirMap ? Array.from(dirMap.entries()) : []; + } else { + // Persistent: derive children from the keys themselves — + // strip the prefix, take the first remaining component, + // dedupe (a subdir contributes one entry per child file). + const prefix = entry.name === "" ? "" : entry.name + "/"; + const seen = new Map(); + for (const k of root.map.keys()) { + if (!k.startsWith(prefix)) continue; + const rest = k.slice(prefix.length); + if (rest === "") continue; + const slash = rest.indexOf("/"); + if (slash < 0) { + if (!seen.has(rest)) seen.set(rest, "file"); + } else { + const name = rest.slice(0, slash); + if (!seen.has(name)) seen.set(name, "dir"); + } + } + // Surface mkdir'd dirs that have no files yet too. + const dirPrefix = entry.name === "" ? "" : entry.name + "/"; + for (const d of persistentDirs) { + if (!d.startsWith(dirPrefix)) continue; + const rest = d.slice(dirPrefix.length); + const slash = rest.indexOf("/"); + const name = slash < 0 ? rest : rest.slice(0, slash); + if (name && !seen.has(name)) seen.set(name, "dir"); + } + children = Array.from(seen.entries()); + } + + const startCookie = typeof cookieArg === "bigint" ? Number(cookieArg) : Number(cookieArg); + let cursor = bufPtr; + let used = 0; + const textEnc = new TextEncoder(); + for (let i = startCookie; i < children.length; ++i) { + const [name, kind] = children[i]; + const nameBytes = textEnc.encode(name); + const recSize = 24 + nameBytes.byteLength; + if (used + 24 > bufLen) break; + const nextCookie = BigInt(i + 1); + dv.setBigUint64(cursor + 0, nextCookie, true); // d_next + dv.setBigUint64(cursor + 8, BigInt(i + 1), true); // d_ino (synthetic) + dv.setUint32(cursor + 16, nameBytes.byteLength, true); // d_namlen + dv.setUint8(cursor + 20, kind === "dir" ? 3 : 4); // d_type + mem.fill(0, cursor + 21, cursor + 24); // padding + const writeName = Math.min(nameBytes.byteLength, bufLen - used - 24); + mem.set(nameBytes.subarray(0, writeName), cursor + 24); + used += 24 + writeName; + cursor += 24 + writeName; + if (writeName < nameBytes.byteLength) break; // truncated name; libc will retry + } + dv.setInt32(bytesUsedPtr, used, true); + return 0; + } fd_renumber() { return 0; } fd_sync() { return 0; } path_filestat_get(dirfd, _dirflags, pathPtr, pathLen, statPtr) { @@ -440,26 +591,34 @@ class Wasi { const dv = new DataView(this.instance.exports.memory.buffer); // Resolve filetype: 4 = regular_file, 3 = directory. The - // preopen mount itself (empty key) is always a directory. Any - // key that is a prefix of another key acts as a directory too - // — `std::filesystem::create_directories` walks parent - // components and stats each one, so reporting them as dirs - // keeps libc happy even though our backing store is flat-keyed. + // preopen mount itself (empty key) is always a directory. On + // the read-only side we built an explicit directory tree at + // startup (readonlyDirs) — anything that key-matches a node is + // a directory. On the rw side, dirs are remembered explicitly + // via persistentDirs (path_create_directory) OR implicitly by + // being a prefix of any persistent file key (so libstdc++'s + // create_directories stat→mkdir→stat loop converges). let filetype; let size = 0n; if (file !== undefined) { filetype = 4; size = BigInt(file.byteLength); - } else if (key === "" || persistentDirs.has(key)) { + } else if (key === "") { filetype = 3; - } else { + } else if (root.kind === "ro" && readonlyDirs.has(key)) { + filetype = 3; + } else if (root.kind === "rw" && persistentDirs.has(key)) { + filetype = 3; + } else if (root.kind === "rw") { const prefix = key + "/"; let isDir = false; for (const k of root.map.keys()) { if (k.startsWith(prefix)) { isDir = true; break; } } - if (!isDir) return 44; // ENOENT + if (!isDir) return 44; filetype = 3; + } else { + return 44; // ENOENT } dv.setBigUint64(statPtr + 0, 0n, true); @@ -544,11 +703,34 @@ class Wasi { const raw = this.#decoder.decode(memory.subarray(pathPtr, pathPtr + pathLen)); const key = resolveKey(root, raw); - const O_CREAT = 0x1; - const O_TRUNC = 0x8; + const O_CREAT = 0x1; + const O_DIRECTORY = 0x2; + const O_TRUNC = 0x8; const create = (oflags & O_CREAT) !== 0; + const wantDir = (oflags & O_DIRECTORY) !== 0; const trunc = (oflags & O_TRUNC) !== 0; + // Directory open path. wasi-libc passes O_DIRECTORY for + // `fs::directory_iterator`; we also treat "empty key + ro" as + // the preopen root. + const isRoDir = root.kind === "ro" && (key === "" || readonlyDirs.has(key)); + const isRwDir = root.kind === "rw" && (key === "" || persistentDirs.has(key) || + (() => { + const prefix = key + "/"; + for (const k of root.map.keys()) if (k.startsWith(prefix)) return true; + return false; + })()); + if (wantDir || (isRoDir && !root.map.has(key)) || (isRwDir && !root.map.has(key))) { + if (!isRoDir && !isRwDir) { + dv.setInt32(openedFdPtr, -1, true); + return 54; // ENOTDIR + } + const fd = this.#nextFd++; + this.#fdTable.set(fd, { rootFd: dirfd, name: key, offset: 0, isDir: true }); + dv.setInt32(openedFdPtr, fd, true); + return 0; + } + let present = root.map.has(key); if (!present) { if (!create || root.kind !== "rw") { @@ -564,7 +746,7 @@ class Wasi { } const fd = this.#nextFd++; - this.#fdTable.set(fd, { rootFd: dirfd, name: key, offset: 0 }); + this.#fdTable.set(fd, { rootFd: dirfd, name: key, offset: 0, isDir: false }); dv.setInt32(openedFdPtr, fd, true); return 0; } @@ -631,9 +813,10 @@ if (!wasmUrl) { // // Manifest entries are relative paths under the bin dir (the layout // `cfg.assets` produces — e.g. "assets/Inter.ttf", -// "mods/3DForts_Base/foo.cmesh"). We fetch each at its full path but key -// the read-only map by basename so `path_open`'s basename-reduction can -// find it regardless of the C++ side's cwd or prefix. +// "mods/3DForts_Base/foo.cmesh"). We fetch each at its full path and +// key the read-only map by that same relative path so directory +// iteration + nested dirs work; path_open's resolveKey normalises +// caller-side variations (leading "./" or "/"). try { const manifestResp = await fetch("files.json"); if (manifestResp.ok) { @@ -644,13 +827,13 @@ try { console.warn(`[wasi] failed to preload ${name}: HTTP ${r.status}`); return; } - const base = name.split(/[\\/]/).filter(Boolean).pop() || name; - readonlyVfs.set(base, new Uint8Array(await r.arrayBuffer())); + readonlyVfs.set(name, new Uint8Array(await r.arrayBuffer())); })); } } catch (e) { console.warn("[wasi] no files.json manifest (or fetch failed); file I/O syscalls will return ENOENT:", e.message); } +roBuildDirTree(); // Hydrate the persistent layer from OPFS. Must complete before main() // runs so any std::ifstream on /persistent/* sees the on-disk bytes.