bvh/hardware_builder.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | #include <algorithm> | ||
| 3 | #include <cstdint> | ||
| 4 | #include <memory> | ||
| 5 | #include <optional> | ||
| 6 | #include <vector> | ||
| 7 | |||
| 8 | #include "chrono_batch_render/bvh/common.h" | ||
| 9 | #include "chrono_batch_render/bvh/kernels.h" | ||
| 10 | #include "chrono_batch_render/bvh/kernels/instances.h" | ||
| 11 | #include "chrono_batch_render/device/accel.h" | ||
| 12 | #include "chrono_batch_render/device/device.h" | ||
| 13 | #include "chrono_batch_render/device/kernels/shared.h" | ||
| 14 | #include "chrono_batch_render/device/program.h" | ||
| 15 | #include "chrono_batch_render/device/queue.h" | ||
| 16 | #include "chrono_batch_render/util/error.h" | ||
| 17 | #include "chrono_batch_render/util/log.h" | ||
| 18 | #include "chrono_batch_render/util/profiler.h" | ||
| 19 | |||
| 20 | namespace chrono::batch_render::bvh { | ||
| 21 | |||
| 22 | using device::Accel; | ||
| 23 | using device::Buffer; | ||
| 24 | using device::BufferRange; | ||
| 25 | using device::BufferUsage; | ||
| 26 | using device::CommandEncoder; | ||
| 27 | using device::Device; | ||
| 28 | using device::Traversal; | ||
| 29 | |||
| 30 | namespace { | ||
| 31 | |||
| 32 | class HardwareBlas final : public Blas { | ||
| 33 | public: | ||
| 34 | static constexpr Traversal kFormat = Traversal::Hardware; | ||
| 35 | |||
| 36 | ✗ | HardwareBlas(Device& device, const Geometry& geometry) | |
| 37 | ✗ | : Blas(kFormat), m_input(ToAccelInput(geometry)), m_buffers(GeometryBuffers(geometry)), | |
| 38 | ✗ | m_accel(device.CreateAccel(m_input, "blas")) {} | |
| 39 | |||
| 40 | ✗ | uint64_t Handle() const override { | |
| 41 | ✗ | return m_accel->Address(); | |
| 42 | } | ||
| 43 | |||
| 44 | ✗ | const device::AccelInput& GetInput() const { | |
| 45 | ✗ | return m_input; | |
| 46 | } | ||
| 47 | |||
| 48 | ✗ | Accel& GetAccel() const { | |
| 49 | ✗ | return *m_accel; | |
| 50 | } | ||
| 51 | |||
| 52 | private: | ||
| 53 | device::AccelInput m_input; | ||
| 54 | std::vector<Ref<Buffer>> m_buffers; | ||
| 55 | Ref<Accel> m_accel; | ||
| 56 | }; | ||
| 57 | |||
| 58 | class HardwareTlas final : public Tlas { | ||
| 59 | public: | ||
| 60 | static constexpr Traversal kFormat = Traversal::Hardware; | ||
| 61 | |||
| 62 | ✗ | HardwareTlas(Device& device, std::span<const uint32_t> capacities) : Tlas(kFormat, capacities) { | |
| 63 | ✗ | const std::vector<uint32_t> offsets = Offsets(capacities); | |
| 64 | ✗ | m_instances = device.CreateBuffer( | |
| 65 | ✗ | {.bytes = std::max<uint64_t>(offsets.back(), 1) * sizeof(InstanceRecord), | |
| 66 | .usage = BufferUsage::AccelBuildInput, | ||
| 67 | ✗ | .label = "tlas instances"}); | |
| 68 | ✗ | m_offsets = device.CreateBuffer( | |
| 69 | ✗ | {.bytes = offsets.size() * sizeof(uint32_t), .label = "tlas offsets"}); | |
| 70 | ✗ | device.GetQueue().Write(*m_offsets, offsets.data()); | |
| 71 | |||
| 72 | ✗ | std::vector<uint64_t> handles; | |
| 73 | ✗ | for (uint32_t w = 0; w < capacities.size(); ++w) { | |
| 74 | ✗ | const device::AccelInstances world = { | |
| 75 | ✗ | .instances = m_instances->Slice(uint64_t(offsets[w]) * sizeof(InstanceRecord), | |
| 76 | ✗ | uint64_t(capacities[w]) * sizeof(InstanceRecord)), | |
| 77 | ✗ | .count = capacities[w]}; | |
| 78 | ✗ | m_accels.push_back(device.CreateAccel(world, "tlas")); | |
| 79 | ✗ | m_worlds.push_back(world); | |
| 80 | ✗ | handles.push_back(m_accels.back()->Address()); | |
| 81 | } | ||
| 82 | |||
| 83 | ✗ | SetHandles(device, std::move(handles)); | |
| 84 | ✗ | } | |
| 85 | |||
| 86 | ✗ | Accel& GetAccel(uint32_t world) const { | |
| 87 | ✗ | return *m_accels[world]; | |
| 88 | } | ||
| 89 | |||
| 90 | ✗ | const device::AccelInstances& GetWorld(uint32_t world) const { | |
| 91 | ✗ | return m_worlds[world]; | |
| 92 | } | ||
| 93 | |||
| 94 | ✗ | Buffer& Instances() const { | |
| 95 | ✗ | return *m_instances; | |
| 96 | } | ||
| 97 | |||
| 98 | ✗ | Buffer& InstanceOffsets() const { | |
| 99 | ✗ | return *m_offsets; | |
| 100 | } | ||
| 101 | |||
| 102 | private: | ||
| 103 | std::vector<Ref<Accel>> m_accels; | ||
| 104 | std::vector<device::AccelInstances> m_worlds; | ||
| 105 | Ref<Buffer> m_instances; | ||
| 106 | Ref<Buffer> m_offsets; | ||
| 107 | }; | ||
| 108 | |||
| 109 | class HardwareBuilder final : public Builder { | ||
| 110 | public: | ||
| 111 | ✗ | explicit HardwareBuilder(Device& device) : Builder(device) {} | |
| 112 | |||
| 113 | ✗ | Traversal Format() const override { | |
| 114 | ✗ | return Traversal::Hardware; | |
| 115 | } | ||
| 116 | |||
| 117 | ✗ | void Prepare() override { | |
| 118 | ✗ | Masking(); | |
| 119 | ✗ | } | |
| 120 | |||
| 121 | ✗ | Ref<Blas> CreateBlas(const Geometry& geometry) override { | |
| 122 | ✗ | return Ref<Blas>(new HardwareBlas(m_device, geometry)); | |
| 123 | } | ||
| 124 | |||
| 125 | ✗ | Ref<Tlas> CreateTlas(std::span<const uint32_t> capacities) override { | |
| 126 | ✗ | return Ref<Tlas>(new HardwareTlas(m_device, capacities)); | |
| 127 | } | ||
| 128 | |||
| 129 | ✗ | void Build(CommandEncoder& encoder, std::span<const Ref<Blas>> blases) override { | |
| 130 | ✗ | BuildBlases("Build", encoder, blases, false); | |
| 131 | ✗ | } | |
| 132 | |||
| 133 | ✗ | void Refit(CommandEncoder& encoder, std::span<const Ref<Blas>> blases) override { | |
| 134 | ✗ | BuildBlases("Refit", encoder, blases, true); | |
| 135 | ✗ | } | |
| 136 | |||
| 137 | ✗ | void Build(CommandEncoder& encoder, Tlas& tlas, BufferRange instances, | |
| 138 | std::optional<BufferRange> counts, std::span<const uint32_t> worlds) override { | ||
| 139 | ✗ | BuildWorlds("Build", encoder, tlas, instances, counts, worlds, false); | |
| 140 | ✗ | } | |
| 141 | |||
| 142 | ✗ | void Refit(CommandEncoder& encoder, Tlas& tlas, BufferRange instances, | |
| 143 | std::optional<BufferRange> counts, std::span<const uint32_t> worlds) override { | ||
| 144 | ✗ | BuildWorlds("Refit", encoder, tlas, instances, counts, worlds, true); | |
| 145 | ✗ | } | |
| 146 | |||
| 147 | ✗ | void SubmitRefit(CommandEncoder&& encoder, Tlas& structure, BufferRange instances, | |
| 148 | std::optional<BufferRange> counts, std::span<const uint32_t> worlds) override { | ||
| 149 | ✗ | const auto& tlas = RequireFormat<HardwareTlas>("Refit", structure); | |
| 150 | ✗ | const std::vector<uint32_t> selected = | |
| 151 | ✗ | PrepareWorlds("Refit", encoder, tlas, instances, counts, worlds, true); | |
| 152 | ✗ | device::Queue& queue = m_device.GetQueue(); | |
| 153 | ✗ | queue.Submit(encoder.Finish()); | |
| 154 | ✗ | const std::vector<BufferRange> scratch = WorldScratches(tlas, selected, true); | |
| 155 | ✗ | std::vector<const Accel*> accels; | |
| 156 | ✗ | for (const uint32_t w : selected) { | |
| 157 | ✗ | accels.push_back(&tlas.GetAccel(w)); | |
| 158 | } | ||
| 159 | ✗ | const uint64_t scratchAddress = scratch.empty() ? 0 : scratch.front().Address(); | |
| 160 | ✗ | if (!m_refit || m_refit->accels != accels || m_refit->scratch != scratchAddress) { | |
| 161 | ✗ | m_refit.reset(); | |
| 162 | ✗ | CommandEncoder recorder = queue.CreateEncoder(); | |
| 163 | ✗ | EmitBuilds(recorder, tlas, selected, scratch, true); | |
| 164 | ✗ | m_refit = | |
| 165 | ✗ | CachedRefit{std::move(accels), scratchAddress, recorder.Finish({.reusable = true})}; | |
| 166 | ✗ | } | |
| 167 | ✗ | queue.Submit(m_refit->commands); | |
| 168 | ✗ | } | |
| 169 | |||
| 170 | private: | ||
| 171 | ✗ | void BuildBlases(const char* what, CommandEncoder& encoder, std::span<const Ref<Blas>> blases, | |
| 172 | bool update) { | ||
| 173 | ✗ | CHRONO_BATCH_RENDER_ZONE("Bvh/HardwareBlases"); | |
| 174 | ✗ | static const profiler::ZoneId buildZone = profiler::ZoneId::Named("Bvh/BlasBuild"); | |
| 175 | ✗ | static const profiler::ZoneId refitZone = profiler::ZoneId::Named("Bvh/BlasRefit"); | |
| 176 | ✗ | std::vector<const HardwareBlas*> hardware; | |
| 177 | ✗ | std::vector<uint64_t> sizes; | |
| 178 | ✗ | for (const Ref<Blas>& blas : blases) { | |
| 179 | ✗ | hardware.push_back(&RequireFormat<HardwareBlas>(what, *blas)); | |
| 180 | ✗ | if (update) { | |
| 181 | ✗ | RequireRefittable(what, hardware.back()->GetInput()); | |
| 182 | } | ||
| 183 | ✗ | const device::AccelSizes& accel = hardware.back()->GetAccel().Sizes(); | |
| 184 | ✗ | sizes.push_back(update ? accel.updateScratch : accel.scratch); | |
| 185 | } | ||
| 186 | ✗ | if (hardware.empty()) { | |
| 187 | ✗ | return; | |
| 188 | } | ||
| 189 | |||
| 190 | ✗ | Log(LogLevel::Debug, "{} {} hardware BLASes", update ? "refitting" : "building", | |
| 191 | ✗ | hardware.size()); | |
| 192 | ✗ | const std::vector<BufferRange> scratch = Scratches(sizes); | |
| 193 | ✗ | const device::ZoneScope zone(encoder, update ? refitZone : buildZone); | |
| 194 | ✗ | encoder.Barrier(); | |
| 195 | ✗ | encoder.Concurrent(true); | |
| 196 | ✗ | for (uint32_t b = 0; b < hardware.size(); ++b) { | |
| 197 | ✗ | encoder.BuildAccel(hardware[b]->GetAccel(), hardware[b]->GetInput(), scratch[b], | |
| 198 | update); | ||
| 199 | } | ||
| 200 | ✗ | encoder.Concurrent(false); | |
| 201 | ✗ | } | |
| 202 | |||
| 203 | ✗ | void BuildWorlds(const char* what, CommandEncoder& encoder, const Tlas& structure, | |
| 204 | BufferRange instances, std::optional<BufferRange> counts, | ||
| 205 | std::span<const uint32_t> worlds, bool update) { | ||
| 206 | ✗ | const auto& tlas = RequireFormat<HardwareTlas>(what, structure); | |
| 207 | ✗ | const std::vector<uint32_t> selected = | |
| 208 | ✗ | PrepareWorlds(what, encoder, tlas, instances, counts, worlds, update); | |
| 209 | ✗ | EmitBuilds(encoder, tlas, selected, WorldScratches(tlas, selected, update), update); | |
| 210 | ✗ | } | |
| 211 | |||
| 212 | ✗ | std::vector<uint32_t> PrepareWorlds(const char* what, CommandEncoder& encoder, | |
| 213 | const HardwareTlas& tlas, BufferRange instances, | ||
| 214 | std::optional<BufferRange> counts, | ||
| 215 | std::span<const uint32_t> worlds, bool update) { | ||
| 216 | ✗ | CHRONO_BATCH_RENDER_ZONE("Bvh/HardwareTlas"); | |
| 217 | ✗ | static const profiler::ZoneId buildZone = profiler::ZoneId::Named("Bvh/TlasBuild"); | |
| 218 | ✗ | static const profiler::ZoneId refitZone = profiler::ZoneId::Named("Bvh/TlasRefit"); | |
| 219 | ✗ | RequireInstances(what, tlas, instances); | |
| 220 | ✗ | std::vector<uint32_t> selected = SelectWorlds(what, tlas, worlds); | |
| 221 | ✗ | const std::vector<uint32_t> offsets = Offsets(tlas.Capacities()); | |
| 222 | ✗ | const uint64_t total = Total(tlas.Capacities()); | |
| 223 | ✗ | Log(LogLevel::Debug, "{} {} of {} hardware TLAS worlds", update ? "refitting" : "building", | |
| 224 | ✗ | selected.size(), tlas.Count()); | |
| 225 | ✗ | const device::ZoneScope zone(encoder, update ? refitZone : buildZone); | |
| 226 | ✗ | if (total > 0) { | |
| 227 | ✗ | encoder.Barrier(); | |
| 228 | ✗ | if (counts) { | |
| 229 | ✗ | encoder.Dispatch(Masking().Entry("MaskInstances"), | |
| 230 | ✗ | MaskInstancesParams{.source = {instances.Address()}, | |
| 231 | ✗ | .destination = {tlas.Instances().Address()}, | |
| 232 | ✗ | .offsets = {tlas.InstanceOffsets().Address()}, | |
| 233 | ✗ | .counts = {counts->Address()}, | |
| 234 | .total = uint32_t(total), | ||
| 235 | ✗ | .worlds = tlas.Count()}, | |
| 236 | ✗ | {.x = device::GroupCount(total, kMaskInstancesThreads)}); | |
| 237 | ✗ | } else if (selected.size() == tlas.Count()) { | |
| 238 | ✗ | const uint64_t bytes = total * sizeof(InstanceRecord); | |
| 239 | ✗ | encoder.Copy(instances.Slice(0, bytes), tlas.Instances().Slice(0, bytes)); | |
| 240 | } else { | ||
| 241 | ✗ | for (const uint32_t w : selected) { | |
| 242 | ✗ | const uint64_t offset = uint64_t(offsets[w]) * sizeof(InstanceRecord); | |
| 243 | ✗ | const uint64_t bytes = uint64_t(tlas.Capacities()[w]) * sizeof(InstanceRecord); | |
| 244 | ✗ | if (bytes > 0) { | |
| 245 | ✗ | encoder.Copy(instances.Slice(offset, bytes), | |
| 246 | ✗ | tlas.Instances().Slice(offset, bytes)); | |
| 247 | } | ||
| 248 | } | ||
| 249 | } | ||
| 250 | } | ||
| 251 | |||
| 252 | ✗ | return selected; | |
| 253 | ✗ | } | |
| 254 | |||
| 255 | ✗ | std::vector<BufferRange> WorldScratches(const HardwareTlas& tlas, | |
| 256 | std::span<const uint32_t> selected, bool update) { | ||
| 257 | ✗ | std::vector<uint64_t> sizes; | |
| 258 | ✗ | for (const uint32_t w : selected) { | |
| 259 | ✗ | const device::AccelSizes& accel = tlas.GetAccel(w).Sizes(); | |
| 260 | ✗ | sizes.push_back(update ? accel.updateScratch : accel.scratch); | |
| 261 | } | ||
| 262 | ✗ | return Scratches(sizes); | |
| 263 | ✗ | } | |
| 264 | |||
| 265 | ✗ | void EmitBuilds(CommandEncoder& encoder, const HardwareTlas& tlas, | |
| 266 | std::span<const uint32_t> selected, std::span<const BufferRange> scratch, | ||
| 267 | bool update) { | ||
| 268 | ✗ | encoder.Barrier(); | |
| 269 | ✗ | encoder.Concurrent(true); | |
| 270 | ✗ | for (uint32_t i = 0; i < selected.size(); ++i) { | |
| 271 | ✗ | encoder.BuildAccel(tlas.GetAccel(selected[i]), tlas.GetWorld(selected[i]), scratch[i], | |
| 272 | update); | ||
| 273 | } | ||
| 274 | ✗ | encoder.Concurrent(false); | |
| 275 | ✗ | } | |
| 276 | |||
| 277 | ✗ | std::vector<BufferRange> Scratches(std::span<const uint64_t> sizes) { | |
| 278 | ✗ | std::vector<uint64_t> offsets; | |
| 279 | ✗ | std::vector<uint64_t> aligned; | |
| 280 | ✗ | uint64_t total = 0; | |
| 281 | ✗ | for (const uint64_t size : sizes) { | |
| 282 | ✗ | offsets.push_back(total); | |
| 283 | ✗ | aligned.push_back(RoundUp(std::max<uint64_t>(size, 1), device::kBufferAlignment)); | |
| 284 | ✗ | total += aligned.back(); | |
| 285 | } | ||
| 286 | |||
| 287 | ✗ | Buffer& buffer = Reserve(m_device, m_scratch, | |
| 288 | ✗ | {.bytes = std::max<uint64_t>(total, 256), | |
| 289 | .usage = BufferUsage::AccelStorage, | ||
| 290 | ✗ | .label = "accel scratch"}); | |
| 291 | |||
| 292 | ✗ | std::vector<BufferRange> ranges; | |
| 293 | ✗ | for (uint32_t i = 0; i < sizes.size(); ++i) { | |
| 294 | ✗ | ranges.push_back(buffer.Slice(offsets[i], aligned[i])); | |
| 295 | } | ||
| 296 | ✗ | return ranges; | |
| 297 | ✗ | } | |
| 298 | |||
| 299 | ✗ | device::Program& Masking() { | |
| 300 | ✗ | if (!m_masking) { | |
| 301 | ✗ | m_masking = | |
| 302 | ✗ | m_device.CreateProgram({.modules = {"chrono_batch_render.bvh.kernels.instances"}, | |
| 303 | .library = &kBvhKernels, | ||
| 304 | .entries = {"MaskInstances"}, | ||
| 305 | ✗ | .label = "tlas instances"}); | |
| 306 | } | ||
| 307 | ✗ | return *m_masking; | |
| 308 | ✗ | } | |
| 309 | |||
| 310 | ✗ | struct CachedRefit { | |
| 311 | std::vector<const Accel*> accels; | ||
| 312 | uint64_t scratch = 0; | ||
| 313 | device::CommandBuffer commands; | ||
| 314 | }; | ||
| 315 | |||
| 316 | Ref<Buffer> m_scratch; | ||
| 317 | Ref<device::Program> m_masking; | ||
| 318 | std::optional<CachedRefit> m_refit; | ||
| 319 | }; | ||
| 320 | |||
| 321 | } | ||
| 322 | |||
| 323 | ✗ | std::unique_ptr<Builder> CreateHardwareBuilder(Device& device) { | |
| 324 | ✗ | return std::make_unique<HardwareBuilder>(device); | |
| 325 | } | ||
| 326 | |||
| 327 | } | ||
| 328 |