render/renderer.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | #include "chrono_batch_render/render/renderer_impl.h" | ||
| 3 | |||
| 4 | #include <algorithm> | ||
| 5 | #include <chrono> | ||
| 6 | #include <cmath> | ||
| 7 | #include <cstring> | ||
| 8 | #include <functional> | ||
| 9 | #include <memory> | ||
| 10 | #include <numeric> | ||
| 11 | #include <optional> | ||
| 12 | #include <ranges> | ||
| 13 | #include <set> | ||
| 14 | #include <span> | ||
| 15 | #include <tuple> | ||
| 16 | #include <utility> | ||
| 17 | #include <vector> | ||
| 18 | |||
| 19 | #include "chrono_batch_render/device/commands.h" | ||
| 20 | #include "chrono_batch_render/material/codegen.h" | ||
| 21 | #include "chrono_batch_render/render/kernels.h" | ||
| 22 | #include "chrono_batch_render/render/kernels/render/render.h" | ||
| 23 | #include "chrono_batch_render/scene/scene_impl.h" | ||
| 24 | #include "chrono_batch_render/scene/schema/columns.h" | ||
| 25 | #include "chrono_batch_render/scene/tables/semantics.h" | ||
| 26 | #include "chrono_batch_render/util/log.h" | ||
| 27 | #include "chrono_batch_render/util/profiler.h" | ||
| 28 | #include "chrono_batch_render/util/text.h" | ||
| 29 | |||
| 30 | namespace chrono::batch_render { | ||
| 31 | |||
| 32 | namespace { | ||
| 33 | |||
| 34 | constexpr uint64_t PointCloud::* kPointFields[] = { | ||
| 35 | &PointCloud::coordinates, &PointCloud::distance, | ||
| 36 | &PointCloud::intensity, &PointCloud::stableId, | ||
| 37 | &PointCloud::emitter, &PointCloud::firing, | ||
| 38 | &PointCloud::echo, &PointCloud::radialVelocity, | ||
| 39 | &PointCloud::instanceId, &PointCloud::radialVelocityVector}; | ||
| 40 | |||
| 41 | } | ||
| 42 | |||
| 43 | 149 | RendererImpl::RendererImpl(Ref<Scene> scene, const RendererDesc& desc) | |
| 44 | 149 | : m_scene(scene ? std::move(scene) : throw Error("A renderer of no scene")), | |
| 45 | 149 | m_maxPixelsPerLaunch(std::clamp<uint64_t>(desc.maxPixelsPerLaunch, 1, ~uint32_t(0))), | |
| 46 | 149 | m_streaming(desc.streaming), | |
| 47 | 149 | m_overlays(desc.overlays ? desc.overlays : Overlays::Create(m_scene->GetDevice())), | |
| 48 | 149 | m_captures(*m_scene) { | |
| 49 | 149 | const SimulationClock& clock = SceneImpl::Get(*m_scene).GetClock(); | |
| 50 | 149 | m_lastTime = clock.end; | |
| 51 | 149 | m_lastResets = clock.resets; | |
| 52 | 149 | const material::MaterialDefinitionSetRef& definitions = | |
| 53 | 149 | SceneImpl::Get(*m_scene).GetMaterials().Definitions(); | |
| 54 | 149 | m_renderProgram = CreateRenderProgram(*definitions, {}); | |
| 55 | 149 | m_renderHash = ProgramHash(*definitions); | |
| 56 | 149 | Log(LogLevel::Debug, "created a renderer: at most {} per launch", | |
| 57 | 149 | Counted(m_maxPixelsPerLaunch, "pixel")); | |
| 58 | 149 | } | |
| 59 | |||
| 60 | 3662 | uint64_t RendererImpl::ProgramHash(const material::MaterialDefinitionSet& definitions) const { | |
| 61 | 3662 | return Fnv1a(SceneImpl::Get(*m_scene).GetCurves().ShapeOptions(), definitions.Hash()); | |
| 62 | } | ||
| 63 | |||
| 64 | Ref<device::Program> | ||
| 65 | 211 | RendererImpl::CreateRenderProgram(const material::MaterialDefinitionSet& definitions, | |
| 66 | std::vector<std::string> entries) const { | ||
| 67 | 211 | auto sources = material::MaterialDispatchSources(definitions); | |
| 68 | 211 | sources[kShapeOptionsFile] = SceneImpl::Get(*m_scene).GetCurves().ShapeOptions(); | |
| 69 | 211 | return m_scene->GetDevice().CreateProgram( | |
| 70 | {.modules = {"chrono_batch_render.render.kernels.render.render"}, | ||
| 71 | .library = &kRenderKernels, | ||
| 72 | 211 | .sources = {sources.begin(), sources.end()}, | |
| 73 | .entries = std::move(entries), | ||
| 74 | .loadEntries = false, | ||
| 75 | 211 | .label = "render", | |
| 76 | 422 | .traversal = SceneImpl::Get(*m_scene).GetBuilder().Format()}); | |
| 77 | 633 | } | |
| 78 | |||
| 79 | 9313 | bool RendererImpl::WaitsForKernels() const { | |
| 80 | 9313 | return m_streaming == Streaming::Blocking || SceneImpl::Get(*m_scene).WaitsForKernels(); | |
| 81 | } | ||
| 82 | |||
| 83 | 3572 | device::Kernel& RendererImpl::RenderEntry(std::string_view name) { | |
| 84 | 3572 | if (!m_usedEntries.contains(name)) { | |
| 85 | 172 | m_usedEntries.emplace(name); | |
| 86 | } | ||
| 87 | 3572 | if (m_pendingProgram) { | |
| 88 | 2903 | device::Kernel& pending = m_pendingProgram->Entry(name); | |
| 89 | 2903 | if (WaitsForKernels() || pending.Ready()) { | |
| 90 | 58 | try { | |
| 91 | 58 | pending.Wait(); | |
| 92 | return pending; | ||
| 93 | ✗ | } catch (const std::exception&) { | |
| 94 | ✗ | } | |
| 95 | } | ||
| 96 | } | ||
| 97 | 3514 | return m_renderProgram->Entry(name); | |
| 98 | } | ||
| 99 | |||
| 100 | 3513 | void RendererImpl::UpdateRenderProgram() { | |
| 101 | 3513 | const material::MaterialDefinitionSetRef& definitions = | |
| 102 | 3513 | SceneImpl::Get(*m_scene).GetMaterials().Definitions(); | |
| 103 | 3513 | const uint64_t hash = ProgramHash(*definitions); | |
| 104 | 3513 | if (hash == m_renderHash || hash == m_failedHash) { | |
| 105 | 559 | m_pendingProgram = nullptr; | |
| 106 | 559 | m_pendingDefinitions = nullptr; | |
| 107 | 2954 | } else if (!m_pendingDefinitions || m_pendingHash != hash) { | |
| 108 | 62 | m_pendingProgram = CreateRenderProgram( | |
| 109 | 64 | *definitions, std::vector<std::string>(m_usedEntries.begin(), m_usedEntries.end())); | |
| 110 | 62 | m_pendingDefinitions = definitions; | |
| 111 | 62 | m_pendingHash = hash; | |
| 112 | 62 | m_pendingStart = std::chrono::steady_clock::now(); | |
| 113 | 124 | Log(LogLevel::Debug, | |
| 114 | "relinking the render kernels for {}{}, keeping the current ones " | ||
| 115 | "until the new ones are ready", | ||
| 116 | 62 | Counted(definitions->Definitions().size(), "generated material definition"), | |
| 117 | 182 | definitions->Planetary() ? " and the planetary materials" : ""); | |
| 118 | } | ||
| 119 | 3513 | if (!m_pendingProgram) { | |
| 120 | 3460 | return; | |
| 121 | } | ||
| 122 | 2954 | if (m_usedEntries.empty()) { | |
| 123 | return; | ||
| 124 | } | ||
| 125 | 2897 | const bool wait = WaitsForKernels(); | |
| 126 | 2897 | try { | |
| 127 | 2952 | for (const std::string& name : m_usedEntries) { | |
| 128 | 2899 | device::Kernel& kernel = m_pendingProgram->Entry(name); | |
| 129 | 2899 | if (wait) { | |
| 130 | 53 | kernel.Wait(); | |
| 131 | 2846 | } else if (!kernel.Ready()) { | |
| 132 | 2844 | return; | |
| 133 | } else { | ||
| 134 | 2 | kernel.Wait(); | |
| 135 | } | ||
| 136 | } | ||
| 137 | ✗ | } catch (const std::exception& error) { | |
| 138 | ✗ | std::string message = error.what(); | |
| 139 | ✗ | for (const material::MaterialDefinition& definition : m_pendingDefinitions->Definitions()) { | |
| 140 | ✗ | message = MapOrigins(std::move(message), definition.origins); | |
| 141 | } | ||
| 142 | ✗ | Log(LogLevel::Error, | |
| 143 | "relinking the render kernels failed, so materials keep their previous look: {}", | ||
| 144 | message); | ||
| 145 | ✗ | m_failedHash = hash; | |
| 146 | ✗ | m_pendingProgram = nullptr; | |
| 147 | ✗ | m_pendingDefinitions = nullptr; | |
| 148 | ✗ | return; | |
| 149 | ✗ | } | |
| 150 | 53 | const double seconds = | |
| 151 | 53 | std::chrono::duration<double>(std::chrono::steady_clock::now() - m_pendingStart).count(); | |
| 152 | 53 | Log(LogLevel::Info, "relinked the render kernels for {}{} in {:.1f} s", | |
| 153 | 53 | Counted(m_pendingDefinitions->Definitions().size(), "generated material definition"), | |
| 154 | 102 | m_pendingDefinitions->Planetary() ? " and the planetary materials" : "", seconds); | |
| 155 | 53 | m_renderProgram = std::move(m_pendingProgram); | |
| 156 | 53 | m_renderHash = hash; | |
| 157 | 53 | m_pendingDefinitions = nullptr; | |
| 158 | } | ||
| 159 | |||
| 160 | 3530 | RenderOutput RendererImpl::Step(std::span<const std::string> products, Stream stream) { | |
| 161 | 3530 | CHRONO_BATCH_RENDER_ZONE("Render/Step"); | |
| 162 | 3530 | const std::lock_guard stepping(m_stepMutex); | |
| 163 | 3530 | SceneUpdate update = | |
| 164 | 3534 | SceneImpl::Get(*m_scene).Update({.finish = m_streaming == Streaming::Blocking}, stream); | |
| 165 | 3530 | StepPlan& plan = PlanProducts(products); | |
| 166 | 3526 | const ProductPlan& planned = *plan.plan; | |
| 167 | |||
| 168 | 3526 | const SimulationClock& clock = update.Clock(); | |
| 169 | 3526 | if (clock.resets != m_lastResets) { | |
| 170 | 2 | m_lastResets = clock.resets; | |
| 171 | 2 | m_lastTime = clock.resetTime; | |
| 172 | 2 | m_captures.Reset(); | |
| 173 | } | ||
| 174 | 3526 | const double start = std::min(m_lastTime, clock.end); | |
| 175 | 3526 | m_captures.Schedule(planned.products, start, clock.end); | |
| 176 | 3526 | m_lastTime = clock.end; | |
| 177 | 3526 | std::set<uint32_t> due; | |
| 178 | 7055 | for (const std::shared_ptr<const ResolvedProduct>& product : planned.products) { | |
| 179 | 3529 | if (m_captures.Due(product->prim)) { | |
| 180 | 3516 | due.insert(product->prim); | |
| 181 | } | ||
| 182 | } | ||
| 183 | |||
| 184 | 3526 | RenderOutput result; | |
| 185 | 3526 | result.simulationStartTime = start; | |
| 186 | 3526 | result.simulationEndTime = clock.end; | |
| 187 | 7055 | for (const std::string& path : planned.paths) { | |
| 188 | 7058 | result.m_products[path] = ProductOutput{path, {}}; | |
| 189 | } | ||
| 190 | 3526 | if (due.empty()) { | |
| 191 | return result; | ||
| 192 | } | ||
| 193 | |||
| 194 | 3513 | if (!planned.taxonomies.empty()) { | |
| 195 | 1 | m_scene->Require(planned.taxonomies); | |
| 196 | } | ||
| 197 | 3513 | update.Refresh(stream); | |
| 198 | 3513 | const Snapshot& snapshot = update.Tables(); | |
| 199 | 3513 | UpdateRenderProgram(); | |
| 200 | 3513 | OutputSet& set = AcquireSet(plan); | |
| 201 | 3513 | const std::vector<SensorPipelineRun> pipelineRuns = PrepareRuns(planned, due); | |
| 202 | 3513 | std::set<uint32_t> ran; | |
| 203 | 3568 | for (const SensorPipelineRun& run : pipelineRuns) { | |
| 204 | 55 | ran.insert(run.products.begin(), run.products.end()); | |
| 205 | } | ||
| 206 | 3513 | SemanticTable& semantics = SceneImpl::Get(*m_scene).GetSemantics(); | |
| 207 | 7029 | for (size_t index = 0; index < planned.products.size(); ++index) { | |
| 208 | 3516 | const ResolvedProduct& product = *planned.products[index]; | |
| 209 | 3516 | if (!due.contains(product.prim)) { | |
| 210 | ✗ | continue; | |
| 211 | } | ||
| 212 | 3516 | FrameOutput& frame = | |
| 213 | 3516 | result.m_products.at(planned.paths[index]) | |
| 214 | 3516 | .frames.emplace_back(FrameOutput{.startTime = result.simulationStartTime, | |
| 215 | 3516 | .endTime = result.simulationEndTime}); | |
| 216 | 7189 | for (const ResolvedVar& var : product.vars) { | |
| 217 | 3673 | if (!var.output.empty() && !ran.contains(uint32_t(index))) { | |
| 218 | ✗ | continue; | |
| 219 | } | ||
| 220 | 3673 | RenderVarOutput& output = frame.renderVars[var.path]; | |
| 221 | 3673 | output.renderVarPath = var.path; | |
| 222 | 3673 | output.sourceName = var.tap ? std::string(kTapSource) | |
| 223 | 3667 | : var.output.empty() | |
| 224 | 3667 | ? std::string(kRenderVarSourceTokens[uint32_t(var.source)]) | |
| 225 | 7346 | : var.output; | |
| 226 | 3673 | if (var.semantic && var.source != RenderVarSource::PointCloud) { | |
| 227 | 1 | const SemanticMap map = semantics.Map(var.source, product.taxonomy); | |
| 228 | 1 | output.idMap = map.map; | |
| 229 | 1 | if (var.shape.empty()) { | |
| 230 | ✗ | output.tensor = map.packed; | |
| 231 | } | ||
| 232 | 1 | } | |
| 233 | } | ||
| 234 | } | ||
| 235 | 7330 | for (const OwnedVar& var : set.vars) { | |
| 236 | 3817 | std::vector<FrameOutput>& frames = result.m_products.at(planned.paths[var.product]).frames; | |
| 237 | 3817 | if (frames.empty()) { | |
| 238 | ✗ | continue; | |
| 239 | } | ||
| 240 | 3817 | const auto found = frames[0].renderVars.find(var.renderVar); | |
| 241 | 3817 | if (found == frames[0].renderVars.end()) { | |
| 242 | ✗ | continue; | |
| 243 | } | ||
| 244 | 3817 | RenderVarOutput& output = found->second; | |
| 245 | 3817 | if (var.channel.empty()) { | |
| 246 | 3614 | output.tensor = var.tensor; | |
| 247 | } else { | ||
| 248 | 203 | output.channels[var.channel] = var.tensor; | |
| 249 | } | ||
| 250 | } | ||
| 251 | 3529 | for (const OwnedOverlay& overlay : set.overlays) { | |
| 252 | 16 | std::vector<FrameOutput>& frames = | |
| 253 | 16 | result.m_products.at(planned.paths[overlay.product]).frames; | |
| 254 | 16 | if (!frames.empty()) { | |
| 255 | 16 | frames[0].overlay = overlay.tensor; | |
| 256 | } | ||
| 257 | } | ||
| 258 | 3513 | result.m_done = Trace(planned, set, due, snapshot, pipelineRuns); | |
| 259 | 3513 | return result; | |
| 260 | 7047 | } | |
| 261 | |||
| 262 | 3530 | RendererImpl::StepPlan& RendererImpl::PlanProducts(std::span<const std::string> products) { | |
| 263 | 3530 | std::shared_ptr<const ProductPlan> planned = SceneImpl::Get(*m_scene).GetPlans().Plan(products); | |
| 264 | 3526 | std::erase_if(m_plans, [&](const StepPlan& plan) { | |
| 265 | 3379 | return plan.plan->id != planned->id && plan.plan->paths == planned->paths; | |
| 266 | }); | ||
| 267 | 3526 | const auto cached = std::ranges::find_if( | |
| 268 | 3363 | m_plans, [&](const StepPlan& plan) { return plan.plan->id == planned->id; }); | |
| 269 | 3526 | if (cached != m_plans.end()) { | |
| 270 | 3360 | m_plans.splice(m_plans.begin(), m_plans, cached); | |
| 271 | 3360 | return m_plans.front(); | |
| 272 | } | ||
| 273 | 166 | uint64_t launches = 0; | |
| 274 | 1494 | for (const uint64_t pixels : planned->pixelCounts) { | |
| 275 | 1328 | launches += (pixels + m_maxPixelsPerLaunch - 1) / m_maxPixelsPerLaunch; | |
| 276 | } | ||
| 277 | 498 | Log(LogLevel::Debug, "planned {}: {}, {} in {}", Counted(planned->paths.size(), "product"), | |
| 278 | 332 | Counted(planned->captureCount, "capture"), Counted(planned->pixelCount, "pixel"), | |
| 279 | 166 | Counted(launches, "launch")); | |
| 280 | 166 | m_plans.push_front({std::move(planned)}); | |
| 281 | 166 | if (m_plans.size() > kCachedPlans) { | |
| 282 | ✗ | m_plans.pop_back(); | |
| 283 | } | ||
| 284 | 166 | return m_plans.front(); | |
| 285 | 3692 | } | |
| 286 | |||
| 287 | 3513 | RendererImpl::OutputSet& RendererImpl::AcquireSet(StepPlan& plan) { | |
| 288 | 3513 | const bool outlined = OverlaysImpl::Get(*m_overlays).Outlined(); | |
| 289 | 3513 | if (plan.set && plan.set->outlined == outlined) { | |
| 290 | 3347 | return *plan.set; | |
| 291 | } | ||
| 292 | |||
| 293 | 166 | CHRONO_BATCH_RENDER_ZONE("Render/AcquireSet"); | |
| 294 | 166 | device::Device& device = m_scene->GetDevice(); | |
| 295 | 166 | const device::BufferUsage usage = | |
| 296 | 166 | device.SharedDevice() ? device::BufferUsage::Interop : device::BufferUsage::None; | |
| 297 | 166 | OutputSet set; | |
| 298 | 166 | set.outlined = outlined; | |
| 299 | 166 | std::map<uint32_t, std::array<uint64_t, kRenderVarSourceCount>> addresses; | |
| 300 | 166 | std::map<uint32_t, uint64_t> primIds; | |
| 301 | 166 | const ProductPlan& planned = *plan.plan; | |
| 302 | 166 | const SensorPipelineTable& pipelines = SceneImpl::Get(*m_scene).GetSensorPipelines(); | |
| 303 | 335 | for (uint32_t index = 0; index < planned.products.size(); ++index) { | |
| 304 | 169 | const std::string& path = planned.paths[index]; | |
| 305 | 169 | const ResolvedProduct& product = *planned.products[index]; | |
| 306 | 169 | std::array<uint64_t, kRenderVarSourceCount>& slots = addresses[product.prim]; | |
| 307 | 425 | for (const ResolvedVar& var : product.vars) { | |
| 308 | 256 | if (var.source == RenderVarSource::PointCloud) { | |
| 309 | 33 | uint64_t& clouds = slots[uint32_t(var.source)]; | |
| 310 | 33 | clouds = AcquireCloud(set, index, path, product, var, clouds); | |
| 311 | 33 | continue; | |
| 312 | 33 | } | |
| 313 | 223 | if (var.shape.empty()) { | |
| 314 | ✗ | continue; | |
| 315 | } | ||
| 316 | 223 | const uint64_t bytes = | |
| 317 | 446 | std::accumulate(var.shape.begin(), var.shape.end(), | |
| 318 | 223 | uint64_t(ElementBytes(var.type)), std::multiplies<>()); | |
| 319 | 223 | const Ref<device::Buffer> buffer = | |
| 320 | 223 | device.CreateBuffer({.bytes = bytes, .usage = usage, .label = var.path}); | |
| 321 | 446 | device::Tensor tensor(buffer, 0, var.type, var.shape); | |
| 322 | 223 | if (var.output.empty()) { | |
| 323 | 200 | slots[uint32_t(var.source)] = tensor.Address(); | |
| 324 | } | ||
| 325 | 446 | set.vars.push_back({index, var.path, std::move(tensor), {}, var.output}); | |
| 326 | 446 | } | |
| 327 | 169 | const std::shared_ptr<const SensorPipelineInstance> instance = | |
| 328 | 169 | product.pipeline == kNoPrim ? nullptr : pipelines.Find(product.pipeline); | |
| 329 | 169 | if (instance && std::ranges::any_of(instance->resolved.values, [](const auto& value) { | |
| 330 | 49 | return value.type == sensor_pipeline::SignalType::ElectronRate; | |
| 331 | 169 | })) { | |
| 332 | 15 | uint64_t& rates = slots[uint32_t(RenderVarSource::ElectronRate)]; | |
| 333 | 15 | if (rates == 0) { | |
| 334 | 4 | const uint64_t bytes = uint64_t(product.sensors.size()) * product.width * | |
| 335 | 4 | product.height * std::max(product.channels, 1u) * | |
| 336 | 4 | sizeof(float); | |
| 337 | 4 | rates = set.rateBuffers | |
| 338 | 4 | .emplace_back( | |
| 339 | 8 | device.CreateBuffer({.bytes = bytes, .label = path + " rates"})) | |
| 340 | 4 | ->Address(); | |
| 341 | } | ||
| 342 | 15 | set.rates[index] = rates; | |
| 343 | } | ||
| 344 | 169 | if (product.overlays.empty()) { | |
| 345 | 160 | continue; | |
| 346 | } | ||
| 347 | 9 | const std::vector<int64_t> shape = {int64_t(product.sensors.size()), product.height, | |
| 348 | 9 | product.width, 4}; | |
| 349 | 9 | const uint64_t pixels = uint64_t(shape[0]) * shape[1] * shape[2]; | |
| 350 | 9 | OwnedOverlay overlay{ | |
| 351 | index, | ||
| 352 | 18 | device::Tensor(device.CreateBuffer( | |
| 353 | 9 | {.bytes = pixels * 4, .usage = usage, .label = path + " overlay"}), | |
| 354 | 27 | 0, device::DType::UInt8, shape)}; | |
| 355 | 9 | uint64_t& depth = slots[uint32_t(RenderVarSource::DistanceToImagePlane)]; | |
| 356 | 9 | if (depth == 0) { | |
| 357 | 9 | overlay.ownDepth = | |
| 358 | 9 | device.CreateBuffer({.bytes = pixels * 4, .label = path + " overlay depth"}); | |
| 359 | 9 | depth = overlay.ownDepth->Address(); | |
| 360 | } | ||
| 361 | 9 | overlay.depth = depth; | |
| 362 | 9 | if (outlined) { | |
| 363 | 4 | overlay.primIds = | |
| 364 | 4 | device.CreateBuffer({.bytes = pixels * 4, .label = path + " overlay prim ids"}); | |
| 365 | 4 | primIds[product.prim] = overlay.primIds->Address(); | |
| 366 | } | ||
| 367 | 9 | set.overlays.push_back(std::move(overlay)); | |
| 368 | 169 | } | |
| 369 | 1494 | for (uint32_t entry = 0; entry < kRenderEntryCount; ++entry) { | |
| 370 | 1328 | if (planned.captures[entry].empty()) { | |
| 371 | 1162 | continue; | |
| 372 | } | ||
| 373 | 166 | std::vector<Capture> captures; | |
| 374 | 359 | for (const PlannedCapture& each : planned.captures[entry]) { | |
| 375 | 193 | Capture& capture = captures.emplace_back(Capture{.plan = each}); | |
| 376 | 193 | const std::array<uint64_t, kRenderVarSourceCount>& slots = addresses[each.product]; | |
| 377 | 193 | std::copy(slots.begin(), slots.end(), std::begin(capture.vars)); | |
| 378 | 193 | const auto found = primIds.find(each.product); | |
| 379 | 193 | capture.primIds = found != primIds.end() ? found->second : 0; | |
| 380 | } | ||
| 381 | 166 | set.captureBuffers[entry] = device.CreateBuffer({.label = "render captures"}, captures); | |
| 382 | 332 | set.captures[entry] = std::move(captures); | |
| 383 | 166 | } | |
| 384 | 166 | Log(LogLevel::Debug, "allocated render outputs for plan {}", planned.id); | |
| 385 | 166 | plan.set = std::move(set); | |
| 386 | 166 | return *plan.set; | |
| 387 | 970 | } | |
| 388 | |||
| 389 | 33 | uint64_t RendererImpl::AcquireCloud(OutputSet& set, uint32_t index, const std::string& path, | |
| 390 | const ResolvedProduct& product, const ResolvedVar& var, | ||
| 391 | uint64_t next) { | ||
| 392 | 33 | device::Device& device = m_scene->GetDevice(); | |
| 393 | 33 | const device::BufferUsage usage = | |
| 394 | 33 | device.SharedDevice() ? device::BufferUsage::Interop : device::BufferUsage::None; | |
| 395 | 33 | const auto sensors = int64_t(product.sensors.size()); | |
| 396 | 33 | const uint32_t returns = std::max(product.returns, 1u); | |
| 397 | 33 | const uint32_t perSensor = product.width * product.height * returns; | |
| 398 | 160 | const auto add = [&](std::string channel, device::DType type, std::vector<int64_t> shape) { | |
| 399 | 254 | const uint64_t bytes = std::accumulate(shape.begin(), shape.end(), | |
| 400 | 127 | uint64_t(ElementBytes(type)), std::multiplies<>()); | |
| 401 | 127 | device::Tensor tensor( | |
| 402 | 254 | device.CreateBuffer( | |
| 403 | ✗ | {.bytes = bytes, .usage = usage, .label = var.path + " " + channel}), | |
| 404 | 381 | 0, type, std::move(shape)); | |
| 405 | 127 | const uint64_t address = tensor.Address(); | |
| 406 | 127 | set.vars.push_back({index, var.path, std::move(tensor), std::move(channel)}); | |
| 407 | 127 | return address; | |
| 408 | 381 | }; | |
| 409 | |||
| 410 | 33 | OwnedCloud owned{product.prim, uint32_t(sensors) * perSensor, index, var.output, | |
| 411 | 33 | var.frame == FrameOfReference::World}; | |
| 412 | 66 | owned.records = device.CreateBuffer( | |
| 413 | 33 | {.bytes = uint64_t(owned.slots) * sizeof(PointRecord), .label = path + " point records"}); | |
| 414 | 33 | owned.flags = | |
| 415 | 33 | device.CreateBuffer({.bytes = uint64_t(owned.slots) * 4, .label = path + " point flags"}); | |
| 416 | 33 | owned.offsets = | |
| 417 | 33 | device.CreateBuffer({.bytes = uint64_t(owned.slots) * 4, .label = path + " point offsets"}); | |
| 418 | 33 | PointCloud cloud{}; | |
| 419 | 33 | cloud.records = owned.records->Address(); | |
| 420 | 33 | cloud.flags = owned.flags->Address(); | |
| 421 | 33 | cloud.offsets = owned.offsets->Address(); | |
| 422 | 33 | cloud.counts = add("Counts", device::DType::UInt32, {sensors}); | |
| 423 | 127 | for (const PointChannel channel : var.channels) { | |
| 424 | 94 | const PointChannelInfo& info = kPointChannels[uint32_t(channel)]; | |
| 425 | 94 | std::vector<int64_t> shape = {sensors, perSensor}; | |
| 426 | 94 | if (info.lanes > 1) { | |
| 427 | 33 | shape.push_back(info.lanes); | |
| 428 | } | ||
| 429 | 188 | cloud.*kPointFields[uint32_t(channel)] = | |
| 430 | 188 | add(std::string(info.name), info.type, std::move(shape)); | |
| 431 | 94 | } | |
| 432 | 33 | cloud.slotsPerSensor = perSensor; | |
| 433 | 33 | cloud.width = product.width; | |
| 434 | 33 | cloud.returns = returns; | |
| 435 | 33 | cloud.sensors = uint32_t(sensors); | |
| 436 | 33 | cloud.world = var.frame == FrameOfReference::World ? 1 : 0; | |
| 437 | 33 | cloud.next = next; | |
| 438 | 66 | owned.cloud = device.CreateBuffer({.label = path + " point cloud"}, std::vector{cloud}); | |
| 439 | 33 | const uint64_t address = owned.cloud->Address(); | |
| 440 | 33 | set.clouds.push_back(std::move(owned)); | |
| 441 | 33 | return address; | |
| 442 | 165 | } | |
| 443 | |||
| 444 | 3513 | device::Operation RendererImpl::Trace(const ProductPlan& plan, OutputSet& set, | |
| 445 | const std::set<uint32_t>& due, const Snapshot& snapshot, | ||
| 446 | std::span<const SensorPipelineRun> runs) { | ||
| 447 | 3513 | static const profiler::ZoneId traceZone = profiler::ZoneId::Named("Render/Trace"); | |
| 448 | 3513 | RenderParams params{}; | |
| 449 | 3513 | params.tables.address = snapshot.tables; | |
| 450 | |||
| 451 | 3513 | device::Queue& queue = m_scene->GetDevice().GetQueue(); | |
| 452 | 3513 | device::CommandEncoder encoder = queue.CreateEncoder(); | |
| 453 | 3513 | encoder.BeginZone(traceZone); | |
| 454 | 3513 | encoder.Concurrent(true); | |
| 455 | 31617 | for (uint32_t entry = 0; entry < kRenderEntryCount; ++entry) { | |
| 456 | 28104 | std::vector<Capture> captures; | |
| 457 | 28104 | uint64_t pixelCount = 0; | |
| 458 | 31669 | for (Capture capture : set.captures[entry]) { | |
| 459 | 3565 | if (due.contains(capture.plan.product)) { | |
| 460 | 3565 | capture.plan.firstPixel = pixelCount; | |
| 461 | 3565 | pixelCount += uint64_t(capture.plan.width) * capture.plan.height; | |
| 462 | 3565 | captures.push_back(capture); | |
| 463 | } | ||
| 464 | } | ||
| 465 | 28104 | if (captures.empty()) { | |
| 466 | 24591 | continue; | |
| 467 | } | ||
| 468 | 3513 | CHRONO_BATCH_RENDER_ZONE("Render/TraceCaptures"); | |
| 469 | 3513 | params.captures.address = | |
| 470 | 3513 | captures.size() == set.captures[entry].size() | |
| 471 | 3513 | ? set.captureBuffers[entry]->Address() | |
| 472 | ✗ | : queue.Upload(captures.data(), captures.size() * sizeof(Capture)).Address(); | |
| 473 | 3513 | params.captureCount = uint32_t(captures.size()); | |
| 474 | 3513 | const uint64_t capturePixels = pixelCount / captures.size(); | |
| 475 | 3513 | const bool uniform = | |
| 476 | 3513 | capturePixels <= UINT32_MAX && std::ranges::all_of(captures, [&](const Capture& c) { | |
| 477 | 3565 | return uint64_t(c.plan.width) * c.plan.height == capturePixels; | |
| 478 | 7026 | }); | |
| 479 | 3513 | params.capturePixels = uniform ? uint32_t(capturePixels) : 0; | |
| 480 | 7026 | for (uint64_t first = 0; first < pixelCount; first += m_maxPixelsPerLaunch) { | |
| 481 | 3513 | params.firstPixel = first; | |
| 482 | 3513 | params.pixelCount = uint32_t(std::min(m_maxPixelsPerLaunch, pixelCount - first)); | |
| 483 | 7026 | encoder.Dispatch(RenderEntry(kRenderEntryNames[entry]), params, | |
| 484 | 3513 | {.x = device::GroupCount(params.pixelCount, kRenderGroup)}); | |
| 485 | } | ||
| 486 | 28104 | } | |
| 487 | 3513 | encoder.Concurrent(false); | |
| 488 | 3513 | std::vector<SensorPipelineRun> traced; | |
| 489 | 3568 | for (const SensorPipelineRun& run : runs) { | |
| 490 | 110 | const bool returns = std::ranges::any_of( | |
| 491 | 55 | run.instance->resolved.values, [](const sensor_pipeline::Value& value) { | |
| 492 | 196 | return value.type == sensor_pipeline::SignalType::Returns; | |
| 493 | }); | ||
| 494 | 55 | const bool read = std::ranges::any_of(set.clouds, [&](const OwnedCloud& cloud) { | |
| 495 | 45 | return !cloud.output.empty() && | |
| 496 | 15 | std::ranges::find(run.products, cloud.index) != run.products.end(); | |
| 497 | }); | ||
| 498 | 55 | if (!returns || read) { | |
| 499 | 55 | traced.push_back(run); | |
| 500 | } | ||
| 501 | } | ||
| 502 | 3513 | RecordRuns(encoder, plan, set, traced, snapshot); | |
| 503 | 3572 | for (const OwnedCloud& cloud : set.clouds) { | |
| 504 | 59 | if (!due.contains(cloud.product)) { | |
| 505 | ✗ | continue; | |
| 506 | } | ||
| 507 | 59 | if (cloud.slots > m_scanCapacity) { | |
| 508 | 18 | m_scanCapacity = cloud.slots; | |
| 509 | 18 | m_scan = std::make_unique<algorithms::Scan>(m_scene->GetDevice(), m_scanCapacity); | |
| 510 | } | ||
| 511 | 59 | m_scan->ExclusiveSum(encoder, *cloud.flags, *cloud.offsets, cloud.slots); | |
| 512 | 59 | ScatterParams scatter{}; | |
| 513 | 59 | scatter.cloud.address = cloud.cloud->Address(); | |
| 514 | 59 | scatter.tables.address = snapshot.tables; | |
| 515 | 59 | scatter.slotCount = cloud.slots; | |
| 516 | 118 | encoder.Dispatch(RenderEntry("RenderScatterPoints"), scatter, | |
| 517 | 59 | {.x = device::GroupCount(cloud.slots, kRenderGroup)}); | |
| 518 | } | ||
| 519 | 3513 | encoder.EndZone(); | |
| 520 | 3513 | std::vector<std::string> stepped; | |
| 521 | 3513 | std::vector<OverlayOutput> views; | |
| 522 | 7029 | for (size_t index = 0; index < plan.products.size(); ++index) { | |
| 523 | 3516 | if (due.contains(plan.products[index]->prim)) { | |
| 524 | 3516 | stepped.push_back(plan.paths[index]); | |
| 525 | } | ||
| 526 | } | ||
| 527 | 3529 | for (const OwnedOverlay& overlay : set.overlays) { | |
| 528 | 16 | const ResolvedProduct& product = *plan.products[overlay.product]; | |
| 529 | 16 | if (!due.contains(product.prim)) { | |
| 530 | ✗ | continue; | |
| 531 | } | ||
| 532 | 16 | OverlayOutput& view = views.emplace_back( | |
| 533 | 16 | OverlayOutput{plan.paths[overlay.product], &product.overlays, &overlay.tensor, {}}); | |
| 534 | 32 | for (uint32_t layer = 0; layer < product.sensors.size(); ++layer) { | |
| 535 | 32 | view.targets.push_back({.camera = product.sensors[layer], | |
| 536 | 16 | .product = product.prim, | |
| 537 | .layer = layer, | ||
| 538 | 16 | .width = product.width, | |
| 539 | 16 | .height = product.height, | |
| 540 | 16 | .missDistance = product.missDistance, | |
| 541 | 16 | .overlay = overlay.tensor.Address(), | |
| 542 | 16 | .depth = overlay.depth, | |
| 543 | 16 | .primIds = overlay.primIds ? overlay.primIds->Address() : 0}); | |
| 544 | } | ||
| 545 | } | ||
| 546 | 3513 | OverlaysImpl& overlays = OverlaysImpl::Get(*m_overlays); | |
| 547 | 3513 | const std::vector<std::shared_ptr<OverlayPickState>> picks = overlays.TakePicks(stepped); | |
| 548 | 3513 | std::vector<OverlayContents> drawn; | |
| 549 | 3513 | if (!views.empty()) { | |
| 550 | 16 | std::vector<OverlayCameraSource> cameraImages; | |
| 551 | 32 | for (const OwnedVar& var : set.vars) { | |
| 552 | 16 | const ResolvedProduct& product = *plan.products[var.product]; | |
| 553 | 16 | const auto source = | |
| 554 | 16 | std::ranges::find_if(product.vars, [&](const ResolvedVar& candidate) { | |
| 555 | 16 | return candidate.path == var.renderVar && | |
| 556 | 16 | candidate.source == RenderVarSource::LdrColor; | |
| 557 | }); | ||
| 558 | 32 | if (!due.contains(product.prim) || source == product.vars.end() || | |
| 559 | 16 | !IsA(product.sensor, SchemaId::Camera)) { | |
| 560 | ✗ | continue; | |
| 561 | } | ||
| 562 | 32 | for (uint32_t layer = 0; layer < product.sensors.size(); ++layer) { | |
| 563 | 32 | cameraImages.push_back({product.sensors[layer], var.tensor.Select(layer)}); | |
| 564 | } | ||
| 565 | } | ||
| 566 | 16 | if (!m_overlayPass) { | |
| 567 | 8 | m_overlayPass.emplace(m_scene->GetDevice()); | |
| 568 | } | ||
| 569 | 16 | drawn = m_overlayPass->Draw(encoder, snapshot, m_scene->WorldCount(), overlays, views, | |
| 570 | 16 | cameraImages, picks); | |
| 571 | 16 | } | |
| 572 | 3513 | const device::Operation done = queue.Submit(encoder.Finish()); | |
| 573 | 3513 | if (!drawn.empty()) { | |
| 574 | 30 | queue.Retire(std::make_shared<std::vector<OverlayContents>>(std::move(drawn)), done); | |
| 575 | } | ||
| 576 | 3513 | OverlayPass::Serve(picks, done); | |
| 577 | 7026 | return done; | |
| 578 | 7042 | } | |
| 579 | |||
| 580 | std::vector<RendererImpl::SensorPipelineRun> | ||
| 581 | 3513 | RendererImpl::PrepareRuns(const ProductPlan& plan, const std::set<uint32_t>& due) { | |
| 582 | 3513 | const SensorPipelineTable& table = SceneImpl::Get(*m_scene).GetSensorPipelines(); | |
| 583 | 3513 | const bool wait = WaitsForKernels(); | |
| 584 | 3513 | std::shared_ptr<const sensor_pipeline::SensorPipelineSet> definitions; | |
| 585 | 3513 | std::vector<SensorPipelineRun> runs; | |
| 586 | 3568 | for (const PlannedSensorPipeline& planned : plan.pipelines) { | |
| 587 | 55 | SensorPipelineRun run{.planned = &planned, .instance = table.Find(planned.pipeline)}; | |
| 588 | 112 | for (const uint32_t product : planned.products) { | |
| 589 | 57 | if (due.contains(plan.products[product]->prim)) { | |
| 590 | 57 | run.products.push_back(product); | |
| 591 | } | ||
| 592 | } | ||
| 593 | 55 | if (run.products.empty() || !run.instance || !run.instance->error.empty()) { | |
| 594 | ✗ | continue; | |
| 595 | } | ||
| 596 | 55 | const uint32_t id = run.instance->definition; | |
| 597 | 55 | if (m_failedPipelines.contains(id)) { | |
| 598 | ✗ | continue; | |
| 599 | } | ||
| 600 | 55 | try { | |
| 601 | 55 | std::shared_ptr<const CompiledSensorPipeline>& compiled = m_pipelinePrograms[id]; | |
| 602 | 55 | if (!compiled) { | |
| 603 | 26 | definitions = definitions ? definitions : table.Definitions(); | |
| 604 | 50 | compiled = CompileSensorPipeline(m_scene->GetDevice(), definitions->Find(id), | |
| 605 | 75 | *table.Registry()); | |
| 606 | } | ||
| 607 | 55 | run.compiled = compiled; | |
| 608 | 126 | for (const sensor_pipeline::GeneratedKernel& step : compiled->generated.kernels) { | |
| 609 | 71 | device::Kernel& kernel = compiled->program->Entry(step.entry); | |
| 610 | 71 | if (!wait && !kernel.Ready()) { | |
| 611 | ✗ | run.compiled = nullptr; | |
| 612 | ✗ | break; | |
| 613 | } | ||
| 614 | 71 | kernel.Wait(); | |
| 615 | } | ||
| 616 | ✗ | } catch (const std::exception& error) { | |
| 617 | ✗ | Log(LogLevel::Error, "compiling {} failed, so its outputs are left out: {}", | |
| 618 | ✗ | run.instance->path, error.what()); | |
| 619 | ✗ | m_failedPipelines.insert(id); | |
| 620 | ✗ | m_pipelinePrograms.erase(id); | |
| 621 | ✗ | continue; | |
| 622 | ✗ | } | |
| 623 | 55 | if (!run.compiled) { | |
| 624 | ✗ | if (m_pendingPipelines.insert(id).second) { | |
| 625 | ✗ | Log(LogLevel::Info, "{} is compiling, so its outputs are left out until it's ready", | |
| 626 | ✗ | run.instance->path); | |
| 627 | } | ||
| 628 | ✗ | continue; | |
| 629 | } | ||
| 630 | 55 | runs.push_back(std::move(run)); | |
| 631 | 55 | } | |
| 632 | 3513 | return runs; | |
| 633 | 3513 | } | |
| 634 | |||
| 635 | 3513 | void RendererImpl::RecordRuns(device::CommandEncoder& encoder, const ProductPlan& plan, | |
| 636 | OutputSet& set, std::span<const SensorPipelineRun> runs, | ||
| 637 | const Snapshot& snapshot) { | ||
| 638 | 3513 | if (runs.empty()) { | |
| 639 | 3460 | return; | |
| 640 | } | ||
| 641 | 53 | device::Device& device = m_scene->GetDevice(); | |
| 642 | 53 | device::Queue& queue = device.GetQueue(); | |
| 643 | 53 | const std::shared_ptr<const sensor_pipeline::NodeRegistry> nodes = | |
| 644 | 53 | SceneImpl::Get(*m_scene).GetSensorPipelines().Registry(); | |
| 645 | 53 | const sensor_pipeline::NodeRegistry& registry = *nodes; | |
| 646 | 53 | const Prims& prims = SceneImpl::Get(*m_scene).GetStructure().GetPrims(); | |
| 647 | 183 | const auto layer = [&](uint32_t product, uint32_t sensor) { | |
| 648 | 130 | const std::vector<uint32_t>& sensors = plan.products[product]->sensors; | |
| 649 | 130 | return uint64_t(std::ranges::find(sensors, sensor) - sensors.begin()); | |
| 650 | 53 | }; | |
| 651 | 55 | const auto scratchBytes = [](const sensor_pipeline::SignalLayout& layout, uint64_t elements) { | |
| 652 | 4 | const uint64_t bytes = (layout.perCapture ? 1 : elements) * layout.channels * sizeof(float); | |
| 653 | 2 | return (bytes + sensor_pipeline::kArenaAlignment - 1) / sensor_pipeline::kArenaAlignment * | |
| 654 | 2 | sensor_pipeline::kArenaAlignment; | |
| 655 | }; | ||
| 656 | 102 | const auto recordBytes = [](uint64_t elements) { | |
| 657 | 49 | const uint64_t bytes = elements * (sizeof(PointRecord) + sizeof(uint32_t)); | |
| 658 | 49 | return (bytes + sensor_pipeline::kArenaAlignment - 1) / sensor_pipeline::kArenaAlignment * | |
| 659 | 49 | sensor_pipeline::kArenaAlignment; | |
| 660 | }; | ||
| 661 | 68 | const auto findCloud = [&](const SensorPipelineRun& run, uint32_t product, | |
| 662 | std::string_view output) { | ||
| 663 | 15 | const OwnedCloud* found = nullptr; | |
| 664 | 30 | for (const OwnedCloud& cloud : set.clouds) { | |
| 665 | 30 | if (cloud.index == product && !cloud.output.empty() && | |
| 666 | 60 | (output.empty() || cloud.output == output) && | |
| 667 | 30 | std::ranges::find(run.products, cloud.index) != run.products.end()) { | |
| 668 | found = &cloud; | ||
| 669 | break; | ||
| 670 | } | ||
| 671 | } | ||
| 672 | 15 | return found; | |
| 673 | 53 | }; | |
| 674 | 159 | const auto read = [&](const SensorPipelineRun& run, const std::string& name) { | |
| 675 | 106 | return std::ranges::any_of(set.vars, [&](const OwnedVar& var) { | |
| 676 | 346 | return var.output == name && | |
| 677 | 106 | std::ranges::find(run.products, var.product) != run.products.end(); | |
| 678 | 106 | }); | |
| 679 | 53 | }; | |
| 680 | |||
| 681 | 57 | struct Placed { | |
| 682 | sensor_pipeline::ArenaLayout arena; | ||
| 683 | uint64_t base = 0; | ||
| 684 | uint64_t scratch = 0; | ||
| 685 | }; | ||
| 686 | |||
| 687 | 53 | std::vector<Placed> placed; | |
| 688 | 53 | uint64_t arenaBytes = 0; | |
| 689 | 108 | for (const SensorPipelineRun& run : runs) { | |
| 690 | 55 | const ResolvedProduct& product = *plan.products[run.products.front()]; | |
| 691 | 55 | const uint64_t elements = | |
| 692 | 55 | uint64_t(product.width) * product.height * std::max(product.returns, 1u); | |
| 693 | 55 | Placed& each = placed.emplace_back( | |
| 694 | 55 | Placed{sensor_pipeline::LayoutArena(run.compiled->definition->segments, product.layouts, | |
| 695 | elements), | ||
| 696 | arenaBytes}); | ||
| 697 | 55 | each.scratch = each.base + each.arena.bytes; | |
| 698 | 55 | arenaBytes = each.scratch; | |
| 699 | 275 | for (const sensor_pipeline::ResolvedNode& node : run.instance->resolved.nodes) { | |
| 700 | 220 | if (registry[node.def].kind == sensor_pipeline::NodeKind::Source && | |
| 701 | 56 | run.instance->resolved.values[node.outputs.front()].type == | |
| 702 | sensor_pipeline::SignalType::Returns) { | ||
| 703 | 15 | arenaBytes += recordBytes(elements); | |
| 704 | } | ||
| 705 | } | ||
| 706 | 128 | for (const sensor_pipeline::ResolvedOutput& output : run.instance->resolved.outputs) { | |
| 707 | 73 | const sensor_pipeline::SignalLayout& layout = product.layouts[output.value]; | |
| 708 | 73 | const sensor_pipeline::SignalType type = | |
| 709 | 73 | run.instance->resolved.values[output.value].type; | |
| 710 | 73 | if (type == sensor_pipeline::SignalType::PointCloud || | |
| 711 | type == sensor_pipeline::SignalType::Returns) { | ||
| 712 | 19 | arenaBytes += recordBytes(elements); | |
| 713 | 54 | } else if (layout.perCapture || !read(run, output.name)) { | |
| 714 | 1 | arenaBytes += scratchBytes(layout, elements); | |
| 715 | } | ||
| 716 | } | ||
| 717 | } | ||
| 718 | 53 | if (!set.arena || set.arena->Bytes() < arenaBytes) { | |
| 719 | 48 | set.arena = device.CreateBuffer( | |
| 720 | 24 | {.bytes = std::max<uint64_t>(arenaBytes, sensor_pipeline::kArenaAlignment), | |
| 721 | 48 | .label = "pipeline arena"}); | |
| 722 | } | ||
| 723 | |||
| 724 | 54 | struct Batch { | |
| 725 | const CompiledSensorPipeline* compiled = nullptr; | ||
| 726 | std::vector<SensorPipelineCapture> captures; | ||
| 727 | std::vector<SensorPipelineValue> values; | ||
| 728 | uint64_t elements = 0; | ||
| 729 | }; | ||
| 730 | |||
| 731 | 53 | std::map<uint32_t, Batch> batches; | |
| 732 | 53 | const uint64_t wordAddress = SceneImpl::Get(*m_scene).GetSensorPipelines().WordsAddress(); | |
| 733 | 53 | std::vector<std::pair<device::BufferRange, device::BufferRange>> copies; | |
| 734 | 53 | std::vector<std::pair<device::BufferRange, device::BufferRange>> sources; | |
| 735 | 108 | for (size_t index = 0; index < runs.size(); ++index) { | |
| 736 | 55 | const SensorPipelineRun& run = runs[index]; | |
| 737 | 55 | const PlannedSensorPipeline& planned = *run.planned; | |
| 738 | 55 | uint32_t source = run.products.front(); | |
| 739 | 70 | for (const OwnedCloud& cloud : set.clouds) { | |
| 740 | 45 | if (!cloud.output.empty() && | |
| 741 | 30 | std::ranges::find(run.products, cloud.index) != run.products.end()) { | |
| 742 | 15 | source = cloud.index; | |
| 743 | 15 | break; | |
| 744 | } | ||
| 745 | } | ||
| 746 | 55 | const ResolvedProduct& product = *plan.products[source]; | |
| 747 | 55 | const sensor_pipeline::SensorPipelineDefinition& definition = *run.compiled->definition; | |
| 748 | 55 | const sensor_pipeline::GeneratedSensorPipeline& generated = run.compiled->generated; | |
| 749 | 55 | const sensor_pipeline::ResolvedSensorPipeline& resolved = run.instance->resolved; | |
| 750 | 55 | const uint32_t returns = std::max(product.returns, 1u); | |
| 751 | 55 | const uint64_t elements = uint64_t(product.width) * product.height * returns; | |
| 752 | 110 | SensorPipelineState& state = m_pipelineStates[std::tuple(planned.pipeline, planned.sensor, | |
| 753 | 55 | product.width, product.height)]; | |
| 754 | 55 | if (!state.state || state.definition != definition.id) { | |
| 755 | 27 | if (state.state) { | |
| 756 | 5 | Log(LogLevel::Debug, "{} restarts its state on {} with a new definition", | |
| 757 | 10 | run.instance->path, m_scene->ByIndex(planned.sensor).Path()); | |
| 758 | } | ||
| 759 | 27 | std::vector<float> initial = definition.initialState; | |
| 760 | 54 | initial.resize(std::max<size_t>(initial.size(), 1)); | |
| 761 | 27 | state = {definition.id, 0, device.CreateBuffer({.label = "pipeline state"}, initial)}; | |
| 762 | 27 | } | |
| 763 | |||
| 764 | 55 | Batch& batch = batches[definition.id]; | |
| 765 | 55 | batch.compiled = run.compiled.get(); | |
| 766 | 55 | const auto firstValue = uint32_t(batch.values.size()); | |
| 767 | 55 | const auto storedCount = uint32_t(definition.segments.stored.size()); | |
| 768 | 55 | uint64_t scratch = placed[index].scratch; | |
| 769 | 196 | for (uint32_t slot = 0; slot < generated.slots.size(); ++slot) { | |
| 770 | 141 | const uint32_t value = generated.slots[slot]; | |
| 771 | 141 | const sensor_pipeline::SignalLayout& layout = product.layouts[value]; | |
| 772 | 141 | SensorPipelineValue entry{0, layout.channels, layout.mosaic ? kSignalMosaic : 0u, 0}; | |
| 773 | 141 | const sensor_pipeline::SignalType type = resolved.values[value].type; | |
| 774 | 160 | const auto bindRecords = [&](const OwnedCloud& cloud, uint64_t first) { | |
| 775 | 19 | entry.address = cloud.records->Address() + first * sizeof(PointRecord); | |
| 776 | 19 | entry.returned = cloud.flags->Address() + first * sizeof(uint32_t); | |
| 777 | 19 | entry.flags |= kSignalRecords | (cloud.world ? kSignalWorld : 0u); | |
| 778 | 160 | }; | |
| 779 | 141 | if (slot < storedCount && type == sensor_pipeline::SignalType::Returns && | |
| 780 | 15 | registry[resolved.nodes[resolved.values[value].node].def].kind == | |
| 781 | sensor_pipeline::NodeKind::Source) { | ||
| 782 | 15 | const OwnedCloud* cloud = findCloud(run, source, {}); | |
| 783 | 15 | if (cloud == nullptr) { | |
| 784 | ✗ | throw Error("{} reads lidar returns, but no point cloud of {} reads it", | |
| 785 | ✗ | run.instance->path, m_scene->ByIndex(product.prim).Path()); | |
| 786 | } | ||
| 787 | 15 | const uint64_t first = layer(source, planned.sensor) * elements; | |
| 788 | 15 | const device::BufferRange records = | |
| 789 | 15 | device::BufferRange(*set.arena).Slice(scratch, elements * sizeof(PointRecord)); | |
| 790 | 15 | const device::BufferRange flags = | |
| 791 | 15 | device::BufferRange(*set.arena) | |
| 792 | 15 | .Slice(scratch + elements * sizeof(PointRecord), | |
| 793 | 15 | elements * sizeof(uint32_t)); | |
| 794 | 15 | scratch += recordBytes(elements); | |
| 795 | 15 | sources.emplace_back( | |
| 796 | 15 | device::BufferRange(*cloud->records) | |
| 797 | 15 | .Slice(first * sizeof(PointRecord), elements * sizeof(PointRecord)), | |
| 798 | records); | ||
| 799 | 15 | sources.emplace_back( | |
| 800 | 15 | device::BufferRange(*cloud->flags) | |
| 801 | 15 | .Slice(first * sizeof(uint32_t), elements * sizeof(uint32_t)), | |
| 802 | flags); | ||
| 803 | 15 | entry.address = records.Address(); | |
| 804 | 15 | entry.returned = flags.Address(); | |
| 805 | 29 | entry.flags |= kSignalRecords | (cloud->world ? kSignalWorld : 0u); | |
| 806 | 126 | } else if (slot >= storedCount && (type == sensor_pipeline::SignalType::PointCloud || | |
| 807 | type == sensor_pipeline::SignalType::Returns)) { | ||
| 808 | 19 | const std::string& name = resolved.outputs[slot - storedCount].name; | |
| 809 | 19 | const OwnedCloud* bound = nullptr; | |
| 810 | 19 | uint64_t boundFirst = 0; | |
| 811 | 39 | for (const uint32_t reader : run.products) { | |
| 812 | 74 | for (const OwnedCloud& cloud : set.clouds) { | |
| 813 | 54 | if (cloud.index != reader || cloud.output != name) { | |
| 814 | 54 | continue; | |
| 815 | } | ||
| 816 | 19 | const uint64_t first = layer(reader, planned.sensor) * elements; | |
| 817 | 19 | if (bound == nullptr) { | |
| 818 | 19 | bindRecords(cloud, first); | |
| 819 | 19 | bound = &cloud; | |
| 820 | 19 | boundFirst = first; | |
| 821 | 19 | continue; | |
| 822 | } | ||
| 823 | ✗ | const auto slice = [&](const Ref<device::Buffer>& buffer, uint64_t at, | |
| 824 | uint64_t stride) { | ||
| 825 | ✗ | return device::BufferRange(*buffer).Slice(at * stride, | |
| 826 | ✗ | elements * stride); | |
| 827 | ✗ | }; | |
| 828 | ✗ | copies.emplace_back(slice(bound->records, boundFirst, sizeof(PointRecord)), | |
| 829 | ✗ | slice(cloud.records, first, sizeof(PointRecord))); | |
| 830 | ✗ | copies.emplace_back(slice(bound->flags, boundFirst, sizeof(uint32_t)), | |
| 831 | ✗ | slice(cloud.flags, first, sizeof(uint32_t))); | |
| 832 | } | ||
| 833 | } | ||
| 834 | 19 | if (!bound) { | |
| 835 | ✗ | entry.address = set.arena->Address() + scratch; | |
| 836 | ✗ | entry.returned = entry.address + elements * sizeof(PointRecord); | |
| 837 | ✗ | entry.flags |= kSignalRecords; | |
| 838 | ✗ | scratch += recordBytes(elements); | |
| 839 | } | ||
| 840 | 107 | } else if (slot < storedCount && | |
| 841 | 53 | registry[resolved.nodes[resolved.values[value].node].def].kind == | |
| 842 | sensor_pipeline::NodeKind::Source) { | ||
| 843 | 41 | const uint32_t stride = std::max(product.channels, 1u); | |
| 844 | 41 | entry.address = set.rates.at(source) + | |
| 845 | 41 | layer(source, planned.sensor) * elements * stride * sizeof(float); | |
| 846 | 41 | if (layout.mosaic) { | |
| 847 | 1 | entry.flags |= kSignalSampled | (stride << kSignalStrideShift); | |
| 848 | } | ||
| 849 | 66 | } else if (slot < storedCount) { | |
| 850 | 12 | entry.address = | |
| 851 | 12 | set.arena->Address() + placed[index].base + placed[index].arena.offsets[value]; | |
| 852 | 12 | entry.flags |= | |
| 853 | 12 | layout.element == sensor_pipeline::ElementType::F16 ? kSignalHalf : 0u; | |
| 854 | } else { | ||
| 855 | 54 | const std::string& name = resolved.outputs[slot - storedCount].name; | |
| 856 | 108 | const uint64_t bytes = | |
| 857 | 54 | (layout.perCapture ? sensor_pipeline::kStatWords : elements * layout.channels) * | |
| 858 | sizeof(float); | ||
| 859 | 54 | std::optional<device::BufferRange> written; | |
| 860 | 54 | if (layout.perCapture || !read(run, name)) { | |
| 861 | 1 | written = device::BufferRange(*set.arena).Slice(scratch, bytes); | |
| 862 | 1 | scratch += scratchBytes(layout, elements); | |
| 863 | } | ||
| 864 | 213 | for (const OwnedVar& var : set.vars) { | |
| 865 | 214 | if (var.output != name || | |
| 866 | 55 | std::ranges::find(run.products, var.product) == run.products.end()) { | |
| 867 | 104 | continue; | |
| 868 | } | ||
| 869 | 55 | const device::BufferRange target = | |
| 870 | 55 | device::BufferRange(var.tensor.GetBuffer()) | |
| 871 | 55 | .Slice(var.tensor.Offset() + layer(var.product, planned.sensor) * bytes, | |
| 872 | bytes); | ||
| 873 | 55 | if (written) { | |
| 874 | 2 | copies.emplace_back(*written, target); | |
| 875 | } else { | ||
| 876 | 53 | written = target; | |
| 877 | } | ||
| 878 | } | ||
| 879 | 54 | entry.address = written->Address(); | |
| 880 | } | ||
| 881 | 141 | batch.values.push_back(entry); | |
| 882 | } | ||
| 883 | |||
| 884 | 55 | const uint32_t detector = prims.FindRow(planned.sensor, SchemaId::DetectorAPI); | |
| 885 | 55 | const uint32_t camera = prims.FindRow(planned.sensor, SchemaId::Camera); | |
| 886 | 55 | batch.captures.push_back( | |
| 887 | 55 | {.firstElement = batch.elements, | |
| 888 | 55 | .words = {wordAddress + uint64_t(run.instance->firstWord) * sizeof(uint32_t)}, | |
| 889 | 55 | .sensor = planned.sensor, | |
| 890 | 55 | .detector = detector == kNoRow ? 0 : detector, | |
| 891 | 55 | .width = product.width * returns, | |
| 892 | 55 | .height = product.height, | |
| 893 | 55 | .capture = state.captures++, | |
| 894 | 55 | .identity = HashPath(m_scene->ByIndex(planned.sensor).Path()).words[0], | |
| 895 | .firstValue = firstValue, | ||
| 896 | 55 | .camera = camera == kNoRow ? 0 : camera, | |
| 897 | 55 | .state = {state.state->Address()}}); | |
| 898 | 55 | batch.elements += elements; | |
| 899 | } | ||
| 900 | |||
| 901 | 53 | encoder.Barrier(); | |
| 902 | 83 | for (const auto& [from, to] : sources) { | |
| 903 | 30 | encoder.Copy(from, to); | |
| 904 | } | ||
| 905 | 53 | if (!sources.empty()) { | |
| 906 | 15 | encoder.Barrier(); | |
| 907 | } | ||
| 908 | 107 | for (auto& [id, batch] : batches) { | |
| 909 | 54 | const SensorPipelineParams params{ | |
| 910 | 54 | .tables = {snapshot.tables}, | |
| 911 | .captures = {queue | ||
| 912 | 54 | .Upload(batch.captures.data(), | |
| 913 | 54 | batch.captures.size() * sizeof(SensorPipelineCapture)) | |
| 914 | 54 | .Address()}, | |
| 915 | .values = {queue | ||
| 916 | 54 | .Upload(batch.values.data(), | |
| 917 | 54 | batch.values.size() * sizeof(SensorPipelineValue)) | |
| 918 | 54 | .Address()}, | |
| 919 | 54 | .elementCount = batch.elements, | |
| 920 | 54 | .captureCount = uint32_t(batch.captures.size())}; | |
| 921 | 54 | RecordSensorPipeline(encoder, *batch.compiled, params, uint32_t(batch.captures.size()), | |
| 922 | batch.elements); | ||
| 923 | } | ||
| 924 | 55 | for (const auto& [from, to] : copies) { | |
| 925 | 2 | encoder.Copy(from, to); | |
| 926 | } | ||
| 927 | 53 | if (!copies.empty()) { | |
| 928 | 1 | encoder.Barrier(); | |
| 929 | } | ||
| 930 | 208 | } | |
| 931 | |||
| 932 | 149 | Renderer::Renderer(Ref<Scene> scene, const RendererDesc& desc) | |
| 933 | 149 | : m_impl(std::make_unique<RendererImpl>(std::move(scene), desc)) {} | |
| 934 | |||
| 935 | 149 | Renderer::~Renderer() = default; | |
| 936 | ✗ | Renderer::Renderer(Renderer&&) noexcept = default; | |
| 937 | ✗ | Renderer& Renderer::operator=(Renderer&&) noexcept = default; | |
| 938 | |||
| 939 | 545 | RenderOutput Renderer::Step(Span<const std::string> products, Stream stream) { | |
| 940 | 545 | return m_impl->Step(products, stream); | |
| 941 | } | ||
| 942 | |||
| 943 | 2985 | RenderOutput Renderer::Step(std::string_view product, Stream stream) { | |
| 944 | 5970 | const std::string products[] = {std::string(product)}; | |
| 945 | 5968 | return m_impl->Step(products, stream); | |
| 946 | 5970 | } | |
| 947 | |||
| 948 | 12 | const Ref<Overlays>& Renderer::GetOverlays() const { | |
| 949 | 12 | return m_impl->m_overlays; | |
| 950 | } | ||
| 951 | |||
| 952 | } | ||
| 953 |