Jorijn van der Graaf
397ff2db51
BuildTLAS - A rebuild for a new instance count or build preference used to create a new acceleration structure and destroy the old one at once, though the command buffer being recorded could still use it (a scene and then a model built in one setup). The acceleration structure now spans its whole storage buffer, and a rebuild that fits builds into it again, so the handle and tlases[i].address stay the same and nothing is destroyed. A TLAS that outgrows its storage gets new storage and a new handle; the old handle goes to the deletion queue ahead of the storage it lives in. It is queued after the scratch is resized, and its address goes with it, so a Resize or create that throws leaves no address of a queued acceleration structure behind. - The barrier before the build also orders earlier traces and compute ray queries of the TLAS, which a refit or a rebuild in place would otherwise overwrite while they read it (BuildTlasWriteBarrier). Its source access mask holds no read bit: one does nothing, but makes synchronization validation 1.4.363 report the second of two builds in one command buffer as a write-after-write hazard. - TlasReadBarrier is now TlasBarrier, shared by both barriers. - Documented: call BuildTLAS once every earlier submission that used tlases[index] has completed, and when the address stays the same. TlasWithBuffer - Moves hand the acceleration structure over, and destroying or assigning over one queues it ahead of its storage. The implicit members dropped the handle over freed storage, or left two owners and a moved-from TLAS that refit through buffers it no longer had. Tests - TLASLifetime: destroying, moving, move-assigning and resetting a TlasWithBuffer. - TLASHighWaterMark skips without ray tracing. It asserts the acceleration structure is rebuilt in place on a shrink or a growth within capacity, that the old handle is queued before its storage on a growth past it, and that builds in one command buffer keep the handle while they fit, queue it once when one outgrows the storage, and free it only on its retire frame. - TLASBuildBarrier checks the barrier before the build too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
347 lines
No EOL
19 KiB
C++
347 lines
No EOL
19 KiB
C++
//SPDX-License-Identifier: LGPL-3.0-only
|
|
//SPDX-FileCopyrightText: Copyright (C) 2026 Catcrafts®
|
|
|
|
module;
|
|
#include <vulkan/vulkan_core.h>
|
|
module Crafter.Graphics:RenderingElement3D_impl;
|
|
import :RenderingElement3D;
|
|
import std;
|
|
|
|
using namespace Crafter;
|
|
|
|
TlasWithBuffer::TlasWithBuffer(TlasWithBuffer&& other)
|
|
: address(std::exchange(other.address, 0))
|
|
, buffer(std::move(other.buffer))
|
|
, accelerationStructure(std::exchange(other.accelerationStructure, VK_NULL_HANDLE))
|
|
, instanceBuffer(std::move(other.instanceBuffer))
|
|
, scratchBuffer(std::move(other.scratchBuffer))
|
|
, metadataBuffer(std::move(other.metadataBuffer))
|
|
, builtInstanceCount(other.builtInstanceCount)
|
|
, builtFlags(other.builtFlags)
|
|
, uploadedVersion(std::move(other.uploadedVersion)) {
|
|
}
|
|
|
|
TlasWithBuffer& TlasWithBuffer::operator=(TlasWithBuffer&& other) {
|
|
if (this == &other) return *this;
|
|
// Ahead of buffer, which it lives in and which queues its own as it
|
|
// takes other's over.
|
|
Device::EnqueueDeletion(accelerationStructure);
|
|
address = std::exchange(other.address, 0);
|
|
buffer = std::move(other.buffer);
|
|
accelerationStructure = std::exchange(other.accelerationStructure, VK_NULL_HANDLE);
|
|
instanceBuffer = std::move(other.instanceBuffer);
|
|
scratchBuffer = std::move(other.scratchBuffer);
|
|
metadataBuffer = std::move(other.metadataBuffer);
|
|
builtInstanceCount = other.builtInstanceCount;
|
|
builtFlags = other.builtFlags;
|
|
uploadedVersion = std::move(other.uploadedVersion);
|
|
return *this;
|
|
}
|
|
|
|
TlasWithBuffer::~TlasWithBuffer() {
|
|
// The body runs before the members' destructors, so the handle is queued
|
|
// ahead of buffer, which it lives in.
|
|
Device::EnqueueDeletion(accelerationStructure);
|
|
}
|
|
|
|
std::vector<RenderingElement3D*> RenderingElement3D::elements;
|
|
|
|
void RenderingElement3D::Add(RenderingElement3D* e) {
|
|
e->indexInElements = static_cast<std::uint32_t>(elements.size());
|
|
elements.push_back(e);
|
|
}
|
|
|
|
void RenderingElement3D::Remove(RenderingElement3D* e) {
|
|
// Idempotent: callers like Builder ghost flow toggle elements in/out
|
|
// and may try to remove an already-removed element.
|
|
std::uint32_t idx = e->indexInElements;
|
|
if (idx == std::numeric_limits<std::uint32_t>::max()) return;
|
|
std::uint32_t last = static_cast<std::uint32_t>(elements.size() - 1);
|
|
if (idx != last) {
|
|
elements[idx] = elements[last];
|
|
elements[idx]->indexInElements = idx;
|
|
}
|
|
elements.pop_back();
|
|
e->indexInElements = std::numeric_limits<std::uint32_t>::max();
|
|
}
|
|
|
|
void RenderingElement3D::BuildTLAS(VkCommandBuffer cmd, std::uint32_t index, RTBuildPreference preference) {
|
|
Device::RequireRayTracing();
|
|
auto& tlas = tlases[index];
|
|
const std::uint32_t primitiveCount = static_cast<std::uint32_t>(elements.size());
|
|
|
|
// ALLOW_UPDATE is always set — BuildTLAS refits in place on later
|
|
// frames. The FAST_TRACE/FAST_BUILD preference is layered on top; the
|
|
// two preference bits are mutually exclusive, so exactly one is chosen.
|
|
// Both bits are baked into the AS at BUILD time and must be identical on
|
|
// every subsequent UPDATE, so a change in `preference` is treated as a
|
|
// topology change below to force a fresh rebuild with the new flags.
|
|
const VkBuildAccelerationStructureFlagsKHR buildFlags =
|
|
VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_UPDATE_BIT_KHR
|
|
| (preference == RTBuildPreference::FastBuild
|
|
? VK_BUILD_ACCELERATION_STRUCTURE_PREFER_FAST_BUILD_BIT_KHR
|
|
: VK_BUILD_ACCELERATION_STRUCTURE_PREFER_FAST_TRACE_BIT_KHR);
|
|
|
|
// Refit (UPDATE) is allowed when the count matches the count this AS
|
|
// was last built for and the build flags are unchanged. A change forces
|
|
// a full rebuild, since a refit must keep the primitive count and the
|
|
// build flags of the build it updates.
|
|
// Refit is dramatically cheaper at scale (millions of instances) — it
|
|
// walks the existing BVH and updates AABBs rather than reconstructing
|
|
// topology.
|
|
const bool topologyChanged =
|
|
tlas.accelerationStructure == VK_NULL_HANDLE
|
|
|| primitiveCount != tlas.builtInstanceCount
|
|
|| buildFlags != tlas.builtFlags;
|
|
|
|
const TlasBarrier writeBarrier = BuildTlasWriteBarrier();
|
|
vkCmdPipelineBarrier(cmd, writeBarrier.srcStage, writeBarrier.dstStage, 0, 1, &writeBarrier.memory, 0, nullptr, 0, nullptr);
|
|
|
|
if (topologyChanged) {
|
|
// Grow the host-visible inputs on a high-water mark rather than
|
|
// resizing to the exact count every topology change. These buffers
|
|
// only need to hold *at least* primitiveCount entries — the AS build
|
|
// reads exactly primitiveCount of them via tlasRangeInfo, and the
|
|
// copy loop below writes [0, primitiveCount) — so an allocation left
|
|
// over from a larger earlier frame is reused as-is. This drops the
|
|
// two host-buffer reallocations on a count decrease (and on an
|
|
// increase that still fits the previous high-water capacity),
|
|
// leaving only the AS build below, which dominates this path
|
|
// regardless; the AS storage and scratch follow a high-water mark
|
|
// of their own.
|
|
//
|
|
// instanceBuffer and metadataBuffer grow in lockstep (always resized
|
|
// together to the same entry count), so one capacity check covers
|
|
// both. size is only meaningful once buffer is non-null, so the
|
|
// null check must short-circuit before the division.
|
|
// STORAGE_BUFFER_BIT is required because the application's compute
|
|
// shaders bind these buffers as storage SSBOs (e.g. to write
|
|
// per-instance transforms directly into the TLAS instance data).
|
|
//
|
|
// instanceBuffer prefers HOST_VISIBLE | DEVICE_LOCAL (BAR/VRAM) so the
|
|
// two per-frame GPU accesses stay on-chip instead of crossing PCIe: the
|
|
// compute shader writes the transform field in place, and the AS build
|
|
// reads the whole instance. The CPU still writes the host-authored
|
|
// fields below — those are write-only, sequential, never-read-back
|
|
// writes, the ideal write-combined BAR workload. DEVICE_LOCAL is a
|
|
// best-effort preference: GetMemoryType falls back to plain
|
|
// HOST_VISIBLE (the previous behaviour) when no combined type exists
|
|
// (no resizable BAR). HOST_COHERENT is intentionally dropped from the
|
|
// required set — the combined type may not be coherent, and the
|
|
// FlushDevice(cmd, ...) below already establishes host->build
|
|
// visibility, gating its flush-skip on the *chosen* type's flags. The
|
|
// single shared buffer is preserved: a whole-buffer copy would clobber
|
|
// the GPU-written transforms, so staging is deliberately avoided.
|
|
//
|
|
// metadataBuffer (#75) gets the same HOST_VISIBLE | prefer-DEVICE_LOCAL
|
|
// upgrade: it is CPU-written every frame (the copy loop below) and read
|
|
// by the ray shaders as a STORAGE_BUFFER every frame, so it has the same
|
|
// per-frame-shader-read-over-PCIe cost. Unlike instanceBuffer there is
|
|
// no GPU co-write, so the direct upgrade is unconditional — but staging
|
|
// is still wrong here: a per-frame-rewritten buffer would pay a stage+
|
|
// copy+barrier every frame, defeating the point. HOST_COHERENT is
|
|
// likewise dropped; the FlushDevice() after the copy loop makes the host
|
|
// writes available before the consuming submit on a non-coherent type
|
|
// (no-op when the chosen type is coherent — the previous behaviour),
|
|
// and the queue submit carries the host-write -> shader-read ordering as
|
|
// it always has.
|
|
//
|
|
// At least one slot each: a TLAS built before anything is in the scene
|
|
// has zero instances, and a zero-size buffer is invalid (NVIDIA fails
|
|
// the allocation). The empty build reads no instance anyway.
|
|
if (tlas.instanceBuffer.buffer == VK_NULL_HANDLE
|
|
|| primitiveCount > tlas.instanceBuffer.size / sizeof(VkAccelerationStructureInstanceKHR)) {
|
|
const std::uint32_t capacity = std::max(primitiveCount, 1u);
|
|
tlas.instanceBuffer.Resize(VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_BUILD_INPUT_READ_ONLY_BIT_KHR | VK_BUFFER_USAGE_STORAGE_BUFFER_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, capacity, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
|
|
tlas.metadataBuffer.Resize(VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, capacity, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT);
|
|
}
|
|
|
|
// A topology change reshuffles which element occupies each slot (a count
|
|
// change, and Remove's swap-and-pop within it) and may have reallocated
|
|
// the host buffers just above — so every slot's prior contents are
|
|
// stale. Clear the per-slot version record (sized to the live count) so
|
|
// the copy loop below re-uploads all primitiveCount slots this frame.
|
|
tlas.uploadedVersion.assign(primitiveCount, 0);
|
|
}
|
|
|
|
// Copy only the slots whose host-authored data changed since this frame's
|
|
// buffers last saw them (#118), tracking the first/last dirty index so the
|
|
// flush below covers just that [first, last] envelope rather than the whole
|
|
// high-water capacity (the envelope may span a few clean slots between
|
|
// sparse dirty ones — flushing a couple of extra entries is far cheaper than
|
|
// the per-slot vkFlushMappedMemoryRanges calls true sub-ranges would need).
|
|
// A slot is dirty when its element advanced past the version
|
|
// recorded for it; an element left at version 0 (never MarkHostDataDirty'd)
|
|
// reads dirty every frame, preserving the pre-#118 copy-every-frame
|
|
// behaviour for callers that don't opt into dirty tracking. Globally-unique
|
|
// versions make this correct under relocation: a slot now holding a
|
|
// different element never matches the version recorded for its previous
|
|
// occupant. The GPU-owned transform is never copied here regardless — the
|
|
// application's compute shader writes it in place earlier in this submission.
|
|
bool anyDirty = false;
|
|
std::uint32_t dirtyFirst = 0;
|
|
std::uint32_t dirtyLast = 0;
|
|
for(std::uint32_t i = 0; i < primitiveCount; i++) {
|
|
const std::uint64_t version = elements[i]->hostDataVersion;
|
|
if (version != 0 && version == tlas.uploadedVersion[i]) {
|
|
continue; // slot already holds this element's current host data
|
|
}
|
|
if (elements[i]->transformOwnedByGpu) {
|
|
// Skip the transform field — the application's compute shader
|
|
// writes it earlier in this submission. Copy everything else.
|
|
auto& dst = tlas.instanceBuffer.value[i];
|
|
const auto& src = elements[i]->instance;
|
|
dst.instanceCustomIndex = src.instanceCustomIndex;
|
|
dst.mask = src.mask;
|
|
dst.instanceShaderBindingTableRecordOffset = src.instanceShaderBindingTableRecordOffset;
|
|
dst.flags = src.flags;
|
|
dst.accelerationStructureReference = src.accelerationStructureReference;
|
|
} else {
|
|
tlas.instanceBuffer.value[i] = elements[i]->instance;
|
|
}
|
|
tlas.metadataBuffer.value[i] = elements[i]->userMetadata;
|
|
tlas.uploadedVersion[i] = version;
|
|
if (!anyDirty) { dirtyFirst = i; anyDirty = true; }
|
|
dirtyLast = i;
|
|
}
|
|
|
|
if (anyDirty) {
|
|
// Flush (and, for the instance buffer, barrier) only the [dirtyFirst,
|
|
// dirtyLast] span actually rewritten this frame. When nothing is dirty
|
|
// there were no host writes at all, so both flushes — and the
|
|
// instanceBuffer HOST->build barrier — are skipped: the AS build reads
|
|
// memory already made visible by the submit that last wrote it, and any
|
|
// GPU-owned transform write is ordered by the application's own
|
|
// compute->build barrier, not this host barrier.
|
|
const std::uint32_t count = dirtyLast - dirtyFirst + 1;
|
|
constexpr VkDeviceSize instStride = sizeof(VkAccelerationStructureInstanceKHR);
|
|
constexpr VkDeviceSize metaStride = sizeof(std::uint32_t);
|
|
|
|
tlas.instanceBuffer.FlushDevice(
|
|
cmd, VK_ACCESS_MEMORY_READ_BIT, VK_PIPELINE_STAGE_ACCELERATION_STRUCTURE_BUILD_BIT_KHR,
|
|
static_cast<VkDeviceSize>(dirtyFirst) * instStride,
|
|
static_cast<VkDeviceSize>(count) * instStride);
|
|
|
|
// Make the per-frame metadata host writes available before the consuming
|
|
// submit. metadataBuffer is not an AS build input (the build never reads
|
|
// it) and has no GPU co-write — only the ray shaders read it, in a later
|
|
// pass — so it needs neither the build-stage barrier instanceBuffer takes
|
|
// above nor a HOST->shader command barrier: the queue submit already
|
|
// orders host writes made available before it against every command in
|
|
// the submission, exactly as it did when this buffer was HOST_COHERENT.
|
|
// This ranged host flush is what now makes those writes available on a
|
|
// non-coherent (BAR/VRAM) type; it self-gates to a no-op on a coherent
|
|
// type (#60), preserving the previous behaviour where the type ends up
|
|
// coherent.
|
|
tlas.metadataBuffer.FlushDevice(
|
|
static_cast<VkDeviceSize>(dirtyFirst) * metaStride,
|
|
static_cast<VkDeviceSize>(count) * metaStride);
|
|
}
|
|
|
|
VkAccelerationStructureGeometryInstancesDataKHR instancesData {
|
|
.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_INSTANCES_DATA_KHR,
|
|
.arrayOfPointers = VK_FALSE,
|
|
.data = {tlas.instanceBuffer.address}
|
|
};
|
|
|
|
VkAccelerationStructureGeometryDataKHR geometryData;
|
|
geometryData.instances = instancesData;
|
|
|
|
VkAccelerationStructureGeometryKHR tlasGeometry {
|
|
.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_KHR,
|
|
.geometryType = VK_GEOMETRY_TYPE_INSTANCES_KHR,
|
|
.geometry = geometryData
|
|
};
|
|
|
|
VkAccelerationStructureBuildGeometryInfoKHR tlasBuildGeometryInfo {
|
|
.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_GEOMETRY_INFO_KHR,
|
|
.type = VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR,
|
|
// ALLOW_UPDATE (required for any subsequent UPDATE-mode refit) plus
|
|
// the caller's FAST_TRACE/FAST_BUILD preference — see buildFlags above.
|
|
.flags = buildFlags,
|
|
.mode = topologyChanged
|
|
? VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR
|
|
: VK_BUILD_ACCELERATION_STRUCTURE_MODE_UPDATE_KHR,
|
|
.geometryCount = 1,
|
|
.pGeometries = &tlasGeometry
|
|
};
|
|
|
|
if (topologyChanged) {
|
|
// Query sizes for the fresh build, and make room for it.
|
|
VkAccelerationStructureBuildSizesInfoKHR tlasBuildSizes {
|
|
.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_SIZES_INFO_KHR
|
|
};
|
|
Device::vkGetAccelerationStructureBuildSizesKHR(
|
|
Device::device,
|
|
VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR,
|
|
&tlasBuildGeometryInfo,
|
|
&primitiveCount,
|
|
&tlasBuildSizes
|
|
);
|
|
|
|
// Scratch buffer must hold at least max(buildScratchSize, updateScratchSize).
|
|
// Sizing for buildScratchSize covers both — refit is always smaller.
|
|
// An empty TLAS may need no scratch at all, but the buffer can't be
|
|
// zero-size.
|
|
tlas.scratchBuffer.Resize(VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, std::max<VkDeviceSize>(tlasBuildSizes.buildScratchSize, 1));
|
|
|
|
// The acceleration structure spans all of its storage, so a build
|
|
// that fits the storage builds into it again: a BUILD may write any
|
|
// acceleration structure created at least as large as it needs. The
|
|
// handle and address then stay the same across count and flag
|
|
// changes, and nothing is queued. Only a TLAS that outgrows its
|
|
// storage gets new storage and a new acceleration structure.
|
|
if (tlas.accelerationStructure == VK_NULL_HANDLE
|
|
|| tlasBuildSizes.accelerationStructureSize > tlas.buffer.capacity) {
|
|
// The old one may still be used earlier in this command buffer,
|
|
// by an earlier build of this index or a trace of it, so it's
|
|
// queued rather than destroyed; ahead of the storage it lives in,
|
|
// which the Resize below queues. Nothing in flight uses it (see
|
|
// BuildTLAS's precondition). The address goes with it, so a
|
|
// Resize or create that throws doesn't leave the address of a
|
|
// queued acceleration structure behind.
|
|
Device::EnqueueDeletion(tlas.accelerationStructure);
|
|
tlas.accelerationStructure = VK_NULL_HANDLE;
|
|
tlas.address = 0;
|
|
|
|
tlas.buffer.Resize(VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_BUILD_INPUT_READ_ONLY_BIT_KHR, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, tlasBuildSizes.accelerationStructureSize);
|
|
|
|
VkAccelerationStructureCreateInfoKHR tlasCreateInfo {
|
|
.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_CREATE_INFO_KHR,
|
|
.buffer = tlas.buffer.buffer,
|
|
.offset = 0,
|
|
.size = tlas.buffer.capacity,
|
|
.type = VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR,
|
|
};
|
|
Device::CheckVkResult(Device::vkCreateAccelerationStructureKHR(Device::device, &tlasCreateInfo, nullptr, &tlas.accelerationStructure));
|
|
|
|
VkAccelerationStructureDeviceAddressInfoKHR addrInfo {
|
|
.sType = VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_DEVICE_ADDRESS_INFO_KHR,
|
|
.accelerationStructure = tlas.accelerationStructure
|
|
};
|
|
tlas.address = Device::vkGetAccelerationStructureDeviceAddressKHR(Device::device, &addrInfo);
|
|
}
|
|
|
|
tlas.builtInstanceCount = primitiveCount;
|
|
tlas.builtFlags = buildFlags;
|
|
}
|
|
|
|
// For UPDATE mode, src == dst (in-place refit). For BUILD, src is
|
|
// VK_NULL_HANDLE and dst is the current handle, kept or fresh.
|
|
tlasBuildGeometryInfo.scratchData.deviceAddress = tlas.scratchBuffer.address;
|
|
tlasBuildGeometryInfo.dstAccelerationStructure = tlas.accelerationStructure;
|
|
tlasBuildGeometryInfo.srcAccelerationStructure =
|
|
topologyChanged ? VK_NULL_HANDLE : tlas.accelerationStructure;
|
|
|
|
VkAccelerationStructureBuildRangeInfoKHR tlasRangeInfo {
|
|
.primitiveCount = primitiveCount,
|
|
.primitiveOffset = 0,
|
|
.firstVertex = 0,
|
|
.transformOffset = 0
|
|
};
|
|
VkAccelerationStructureBuildRangeInfoKHR* tlasRangeInfoPP = &tlasRangeInfo;
|
|
Device::vkCmdBuildAccelerationStructuresKHR(cmd, 1, &tlasBuildGeometryInfo, &tlasRangeInfoPP);
|
|
|
|
const TlasBarrier readBarrier = BuildTlasReadBarrier();
|
|
vkCmdPipelineBarrier(cmd, readBarrier.srcStage, readBarrier.dstStage, 0, 1, &readBarrier.memory, 0, nullptr, 0, nullptr);
|
|
} |