GCC Code Coverage Report

chrono_batch_render/
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
414 / 435
95.2%
Branches:
377 / 438
86.1%
Functions:
24 / 24
100.0%
scene/tables/instance_layout.cpp
Line Branch Exec Source
1 // SPDX-License-Identifier: BSD-3-Clause
2 #include "chrono_batch_render/scene/tables/instances.h"
3
4 #include <algorithm>
5 #include <functional>
6 #include <limits>
7 #include <map>
8
9 #include "chrono_batch_render/device/commands.h"
10 #include "chrono_batch_render/device/kernels/shared.h"
11 #include "chrono_batch_render/device/queue.h"
12 #include "chrono_batch_render/scene/schema/columns.h"
13 #include "chrono_batch_render/scene/tables/kernels/instances.h"
14 #include "chrono_batch_render/stage/publish/change_set.h"
15 #include "chrono_batch_render/util/error.h"
16 #include "chrono_batch_render/util/log.h"
17 #include "chrono_batch_render/util/profiler.h"
18 #include "chrono_batch_render/util/span.h"
19
20 namespace chrono::batch_render {
21
22 namespace {
23
24 12319 bool Drawable(SchemaId schema) {
25
2/2
✓ Branch 1 taken 1521 times.
✓ Branch 2 taken 10798 times.
 12319 return IsA(schema, SchemaId::Gprim) || IsEmitter(uint32_t(schema));
26 }
27
28 }
29
30 580 bool InstanceTable::ShowsGlobals(uint32_t tlas) const {
31 580 const auto composed = m_composed.find(tlas);
32
3/4
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 579 times.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 580 return composed == m_composed.end() || composed->second.global;
33 }
34
35 560 std::vector<uint32_t> InstanceTable::RegionItems(uint32_t tlas) const {
36 560 std::vector<uint32_t> items = m_blockItems[tlas];
37
2/2
✓ Branch 0 taken 114 times.
✓ Branch 1 taken 446 times.
 560 if (tlas < m_worlds) {
38
1/1
✓ Branch 1 taken 114 times.
 114 items.insert(items.end(), m_own[tlas].begin(), m_own[tlas].end());
39 }
40
2/2
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 559 times.
 560 if (const auto composed = m_composed.find(tlas); composed != m_composed.end()) {
41
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 1 time.
 2 for (const uint32_t world : composed->second.worlds) {
42
2/4
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 if (world < m_worlds && world != tlas) {
43
1/1
✓ Branch 1 taken 1 time.
 1 items.insert(items.end(), m_own[world].begin(), m_own[world].end());
44 }
45 }
46 }
47
1/2
✓ Branch 1 taken 560 times.
✗ Branch 2 not taken.
 560 if (ShowsGlobals(tlas)) {
48
1/1
✓ Branch 1 taken 560 times.
 560 items.insert(items.end(), m_globals.begin(), m_globals.end());
49 }
50 560 return items;
51 ✗ }
52
53 311 void InstanceTable::Compose() {
54 311 m_compositions = m_structure.CompositionVersion();
55 311 m_composed.clear();
56 311 m_shownIn.clear();
57
3/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 1 time.
✓ Branch 3 taken 311 times.
 312 for (const auto& [world, desc] : m_structure.Compositions()) {
58
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 if (world >= m_worlds) {
59 ✗ continue;
60 }
61 1 m_composed[world] = desc;
62
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 1 time.
 2 for (const uint32_t shown : desc.worlds) {
63
2/4
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 if (shown < m_worlds && shown != world) {
64 1 m_shownIn[shown].insert(world);
65 }
66 }
67 }
68 311 }
69
70 319 std::vector<InstanceTable::Instancer> InstanceTable::GatherInstancers() const {
71 319 const Prims& prims = m_structure.GetPrims();
72 319 ColumnStore& store = m_structure.GetColumnStore();
73 343 const auto targets = [&](ColumnRef column, uint32_t row) {
74 24 const ArrayRef array = store.Array(column, row);
75 24 std::vector<uint32_t> found =
76
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 24 times.
 24 array.address == 0 ? std::vector<uint32_t>{}
77
3/5
✓ Branch 1 taken 24 times.
✓ Branch 4 taken 24 times.
✓ Branch 7 taken 24 times.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
 24 : Items<uint32_t>(store.Arrays(ColumnType::PrimArray).Read(array));
78
2/2
✓ Branch 0 taken 24 times.
✓ Branch 1 taken 24 times.
 48 for (uint32_t& prim : found) {
79
3/5
✓ Branch 0 taken 24 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 24 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 24 times.
 24 if (prim != kNoPrim && !prims.Alive(prim)) {
80 ✗ prim = kNoPrim;
81 }
82 }
83 24 return found;
84 319 };
85 319 std::vector<Instancer> instancers;
86
2/2
✓ Branch 0 taken 7402 times.
✓ Branch 1 taken 319 times.
 7721 for (uint32_t prim = 0; prim < prims.Size(); ++prim) {
87
3/3
✓ Branch 1 taken 7402 times.
✓ Branch 3 taken 256 times.
✓ Branch 4 taken 7146 times.
 7402 if (!prims.Alive(prim)) {
88 7378 continue;
89 }
90
1/1
✓ Branch 1 taken 7146 times.
 7146 const PrimRecord& record = prims.Record(prim);
91 7146 Instancer instancer{.prim = prim, .world = record.world};
92
2/2
✓ Branch 0 taken 23 times.
✓ Branch 1 taken 7123 times.
 7146 if (record.schema == SchemaId::PointInstancer) {
93
1/1
✓ Branch 1 taken 23 times.
 23 const uint32_t row = prims.Row(prim, SchemaId::PointInstancer);
94
1/1
✓ Branch 1 taken 23 times.
 23 instancer.count = store.Array(kPointInstancerPositions, row).count;
95
1/1
✓ Branch 1 taken 23 times.
 46 instancer.roots = targets(kPointInstancerPrototypes, row);
96
3/3
✓ Branch 1 taken 7123 times.
✓ Branch 3 taken 1 time.
✓ Branch 4 taken 7122 times.
 7123 } else if (prims.HasApi(prim, SchemaId::PrototypeAPI)) {
97 1 instancer.native = true;
98 1 instancer.count = 1;
99 1 instancer.roots =
100
2/2
✓ Branch 1 taken 1 time.
✓ Branch 4 taken 1 time.
 2 targets(kPrototypeAPIPrototype, prims.Row(prim, SchemaId::PrototypeAPI));
101
2/3
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
✓ Branch 3 taken 1 time.
 1 instancer.roots.resize(std::min<size_t>(instancer.roots.size(), 1));
102 } else {
103 7122 continue;
104 }
105
1/1
✓ Branch 1 taken 24 times.
 24 instancers.push_back(std::move(instancer));
106 7146 }
107 319 return instancers;
108 ✗ }
109
110 5214 bool InstanceTable::Member(uint32_t prim) const {
111 5214 const Prims& prims = m_structure.GetPrims();
112
2/2
✓ Branch 1 taken 5262 times.
✓ Branch 2 taken 5214 times.
 10476 for (uint32_t at = prim; at != kNoPrim; at = prims.Record(at).parent) {
113
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 5262 times.
 5262 if (m_roots.contains(at)) {
114 ✗ return true;
115 }
116 }
117 5214 return false;
118 }
119
120 311 void InstanceTable::Relayout() {
121
4/5
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 306 times.
✓ Branch 3 taken 5 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 5 times.
 311 CHRONO_BATCH_RENDER_ZONE("Instances/Relayout");
122
1/1
✓ Branch 1 taken 311 times.
 311 const Prims& prims = m_structure.GetPrims();
123
1/1
✓ Branch 1 taken 311 times.
 311 device::Device& device = m_structure.GetColumnStore().GetDevice();
124
1/1
✓ Branch 1 taken 311 times.
 311 m_worlds = prims.WorldCount();
125
1/1
✓ Branch 1 taken 311 times.
 311 m_own.assign(m_worlds, {});
126 311 m_globals.clear();
127 311 m_origins.clear();
128 311 m_products.clear();
129
1/1
✓ Branch 1 taken 311 times.
 311 m_instancers = GatherInstancers();
130 311 m_roots.clear();
131
2/2
✓ Branch 0 taken 16 times.
✓ Branch 1 taken 311 times.
 327 for (const Instancer& instancer : m_instancers) {
132
2/2
✓ Branch 0 taken 16 times.
✓ Branch 1 taken 16 times.
 32 for (const uint32_t root : instancer.roots) {
133
1/2
✓ Branch 0 taken 16 times.
✗ Branch 1 not taken.
 16 if (root != kNoPrim) {
134
1/1
✓ Branch 1 taken 16 times.
 16 m_roots.insert(root);
135 }
136 }
137 }
138 311 m_members.clear();
139 311 m_drawables.clear();
140 311 m_nested.clear();
141 311 std::set<uint32_t> instancers;
142
2/2
✓ Branch 0 taken 16 times.
✓ Branch 1 taken 311 times.
 327 for (const Instancer& instancer : m_instancers) {
143
1/1
✓ Branch 1 taken 16 times.
 16 instancers.insert(instancer.prim);
144 }
145
1/1
✓ Branch 1 taken 311 times.
 311 Compose();
146
3/3
✓ Branch 0 taken 258 times.
✓ Branch 1 taken 53 times.
✓ Branch 3 taken 311 times.
 569 std::vector<uint32_t> origins(std::max(m_worlds, 1u), kNoRow);
147
2/2
✓ Branch 0 taken 7361 times.
✓ Branch 1 taken 311 times.
 7672 for (uint32_t prim = 0; prim < prims.Size(); ++prim) {
148
3/3
✓ Branch 1 taken 7361 times.
✓ Branch 3 taken 256 times.
✓ Branch 4 taken 7105 times.
 7361 if (!prims.Alive(prim)) {
149 256 continue;
150 }
151
1/1
✓ Branch 1 taken 7105 times.
 7105 const PrimRecord& record = prims.Record(prim);
152 7105 uint32_t root = kNoPrim;
153
5/5
✓ Branch 1 taken 8222 times.
✓ Branch 3 taken 8248 times.
✓ Branch 4 taken 7079 times.
✓ Branch 5 taken 8222 times.
✓ Branch 6 taken 26 times.
 15327 for (uint32_t at = prim; at != kNoPrim && root == kNoPrim; at = prims.Record(at).parent) {
154
2/2
✓ Branch 1 taken 26 times.
✓ Branch 2 taken 8196 times.
 8222 if (m_roots.contains(at)) {
155 26 root = at;
156 }
157 }
158
2/2
✓ Branch 0 taken 26 times.
✓ Branch 1 taken 7079 times.
 7105 if (root != kNoPrim) {
159
1/1
✓ Branch 1 taken 26 times.
 26 m_members.insert(prim);
160
3/3
✓ Branch 1 taken 26 times.
✓ Branch 3 taken 19 times.
✓ Branch 4 taken 7 times.
 26 if (Drawable(record.schema)) {
161
2/2
✓ Branch 1 taken 19 times.
✓ Branch 4 taken 19 times.
 19 m_drawables[root].push_back(prim);
162 }
163
2/2
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 25 times.
 26 if (instancers.contains(prim)) {
164
2/2
✓ Branch 1 taken 1 time.
✓ Branch 4 taken 1 time.
 1 m_nested[root].push_back(prim);
165 }
166
3/3
✓ Branch 1 taken 7079 times.
✓ Branch 3 taken 5699 times.
✓ Branch 4 taken 1380 times.
 7079 } else if (Drawable(record.schema)) {
167
3/3
✓ Branch 0 taken 157 times.
✓ Branch 1 taken 5542 times.
✓ Branch 3 taken 5699 times.
 5699 (record.world == kGlobalWorld ? m_globals : m_own[record.world]).push_back(prim);
168 }
169
3/3
✓ Branch 1 taken 7105 times.
✓ Branch 3 taken 107 times.
✓ Branch 4 taken 6998 times.
 7105 if (IsA(record.schema, SchemaId::World)) {
170
2/2
✓ Branch 1 taken 107 times.
✓ Branch 4 taken 107 times.
 107 origins[record.world] = prims.Row(prim, SchemaId::Xformable);
171
1/1
✓ Branch 1 taken 107 times.
 107 m_origins.insert(prim);
172 }
173
2/2
✓ Branch 0 taken 189 times.
✓ Branch 1 taken 6916 times.
 7105 if (record.schema == SchemaId::RenderProduct) {
174
1/1
✓ Branch 1 taken 189 times.
 189 m_products.push_back(prim);
175 }
176 }
177 311 m_productBuffer =
178
1/1
✓ Branch 1 taken 311 times.
 622 device.CreateBuffer({.label = "render products"},
179 std::span<const uint32_t>(
180
7/7
✓ Branch 0 taken 145 times.
✓ Branch 1 taken 166 times.
✓ Branch 3 taken 145 times.
✓ Branch 6 taken 166 times.
✓ Branch 8 taken 11 times.
✓ Branch 9 taken 300 times.
✓ Branch 11 taken 311 times.
 933 m_products.empty() ? std::vector<uint32_t>{kNoPrim} : m_products));
181
182
1/1
✓ Branch 1 taken 311 times.
 311 BuildInstancing();
183
184 311 std::vector<std::vector<uint32_t>> lists;
185
2/2
✓ Branch 0 taken 418 times.
✓ Branch 1 taken 311 times.
 729 for (uint32_t tlas = 0; tlas <= m_worlds; ++tlas) {
186
1/1
✓ Branch 1 taken 418 times.
 836 lists.push_back(RegionItems(tlas));
187 }
188
1/1
✓ Branch 1 taken 311 times.
 311 m_regions.Assign(std::move(lists));
189 311 std::vector<uint32_t> capacities;
190
2/2
✓ Branch 0 taken 418 times.
✓ Branch 1 taken 311 times.
 729 for (uint32_t tlas = 0; tlas <= m_worlds; ++tlas) {
191
1/1
✓ Branch 1 taken 418 times.
 418 capacities.push_back(m_regions.Capacity(tlas));
192
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 418 times.
 418 if (capacities.back() > kInstanceCustomMask + 1) {
193 ✗ throw Error("{} instances in one TLAS; a TLAS holds at most {}", capacities.back(),
194
0/1
✗ Branch 2 not taken.
 ✗ kInstanceCustomMask + 1);
195 }
196 }
197
198
3/3
✓ Branch 1 taken 311 times.
✓ Branch 3 taken 11 times.
✓ Branch 4 taken 300 times.
 311 m_tlas = m_builder.CreateTlas(capacities);
199
1/1
✓ Branch 1 taken 311 times.
 311 const uint32_t slots = uint32_t(m_regions.Items().size());
200
1/1
✓ Branch 1 taken 311 times.
 311 m_records.Allocate(device, slots);
201
1/1
✓ Branch 1 taken 311 times.
 311 m_shapes.Allocate(device, slots);
202
1/1
✓ Branch 1 taken 311 times.
 311 m_hits.Allocate(device, slots);
203
1/1
✓ Branch 1 taken 311 times.
 311 m_slotWorlds.Allocate(device, slots);
204
1/1
✓ Branch 1 taken 311 times.
 311 m_renderOrigins.Allocate(device, m_worlds + 1);
205
1/1
✓ Branch 1 taken 311 times.
 311 device::CommandEncoder clear = device.GetQueue().CreateEncoder();
206 311 for (const Ref<device::Buffer>& buffer :
207 {m_records.Buffer(), m_shapes.Buffer(), m_hits.Buffer(), m_slotWorlds.Buffer(),
208
3/6
✓ Branch 5 taken 1555 times.
✓ Branch 6 taken 311 times.
✓ Branch 7 taken 1555 times.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
 4976 m_renderOrigins.Buffer()}) {
209
1/1
✓ Branch 2 taken 1555 times.
 1555 clear.Fill(*buffer);
210 1866 }
211
2/2
✓ Branch 1 taken 311 times.
✓ Branch 4 taken 311 times.
 311 device.GetQueue().Submit(clear.Finish());
212
213
1/1
✓ Branch 1 taken 311 times.
 311 std::vector<uint32_t> words(sizeof(InstanceLayout) / sizeof(uint32_t));
214 2177 const auto append = [&](std::span<const uint32_t> values) {
215 1866 const uint64_t at = words.size() * sizeof(uint32_t);
216 1866 words.insert(words.end(), values.begin(), values.end());
217 1866 return at;
218 311 };
219
1/1
✓ Branch 1 taken 311 times.
 311 const uint64_t firstInstancesStart = append(m_regions.Starts());
220
1/1
✓ Branch 1 taken 311 times.
 311 const uint64_t countsStart = append(m_regions.Counts());
221
1/1
✓ Branch 1 taken 311 times.
 311 const uint64_t primsStart = append(m_regions.Items());
222
1/1
✓ Branch 1 taken 311 times.
 311 const uint64_t originsStart = append(origins);
223
1/1
✓ Branch 1 taken 311 times.
 311 std::vector<uint32_t> bases(m_worlds + 1, kNoRow);
224 311 std::vector<uint32_t> offsets;
225
3/3
✓ Branch 1 taken 1 time.
✓ Branch 3 taken 1 time.
✓ Branch 4 taken 311 times.
 312 for (const auto& [world, desc] : m_composed) {
226
1/1
✓ Branch 1 taken 1 time.
 1 bases[world] = uint32_t(offsets.size() / 3);
227 1 const float hidden = std::numeric_limits<float>::quiet_NaN();
228
1/1
✓ Branch 1 taken 1 time.
 1 std::vector<Float3> shifts(m_worlds, Float3{hidden, hidden, hidden});
229
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 1 time.
 2 for (size_t at = 0; at < desc.worlds.size(); ++at) {
230
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 if (desc.worlds[at] < m_worlds) {
231
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 const Double3 offset = at < desc.offsets.size() ? desc.offsets[at] : Double3{};
232 1 shifts[desc.worlds[at]] = {float(offset.x), float(offset.y), float(offset.z)};
233 }
234 }
235
1/1
✓ Branch 1 taken 1 time.
 1 const auto* words32 = reinterpret_cast<const uint32_t*>(shifts.data());
236
1/1
✓ Branch 1 taken 1 time.
 1 offsets.insert(offsets.end(), words32, words32 + shifts.size() * 3);
237 1 }
238
1/1
✓ Branch 1 taken 311 times.
 311 const uint64_t basesStart = append(bases);
239
1/1
✓ Branch 1 taken 311 times.
 311 const uint64_t offsetsStart = append(offsets);
240 311 m_layout =
241
4/4
✓ Branch 1 taken 311 times.
✓ Branch 4 taken 311 times.
✓ Branch 6 taken 11 times.
✓ Branch 7 taken 300 times.
 322 device.CreateBuffer({.bytes = words.size() * sizeof(uint32_t), .label = "instance layout"});
242 622 device.GetQueue().Write(*m_layout, words.data());
243
1/1
✓ Branch 1 taken 311 times.
 311 const uint64_t layout = m_layout->Address();
244 311 m_header.firstInstances.address = layout + firstInstancesStart;
245 311 m_header.instanceCounts.address = layout + countsStart;
246 311 m_header.instancePrims.address = layout + primsStart;
247 311 m_header.worldOrigins.address = layout + originsStart;
248 311 m_header.offsetBases.address = layout + basesStart;
249 311 m_header.offsets.address = layout + offsetsStart;
250
2/2
✓ Branch 1 taken 311 times.
✓ Branch 4 taken 311 times.
 933 m_viewers = device.CreateBuffer(
251
1/1
✓ Branch 1 taken 311 times.
 311 {.label = "viewers"},
252
4/4
✓ Branch 1 taken 311 times.
✓ Branch 3 taken 11 times.
✓ Branch 4 taken 300 times.
✓ Branch 6 taken 311 times.
 933 std::span<const uint32_t>(std::vector<uint32_t>(m_worlds + 1, kNoPrim)));
253
1/1
✓ Branch 1 taken 311 times.
 311 WriteLayoutHeader();
254
255
1/1
✓ Branch 1 taken 311 times.
 311 m_refits.assign(m_worlds + 1, 0);
256 311 ++m_relayouts;
257 311 Log(LogLevel::Debug,
258 "laid out {} TLASes with room for {} instances, {} global prims in each and {} "
259 "instancers",
260
1/1
✓ Branch 1 taken 311 times.
 311 m_worlds + 1, m_regions.Items().size(), m_globals.size(), m_instancers.size());
261 2488 }
262
263 317 void InstanceTable::BuildInstancing() {
264 317 const Prims& prims = m_structure.GetPrims();
265 317 ColumnStore& store = m_structure.GetColumnStore();
266
267 338 struct Placed {
268 uint32_t first = 0;
269 uint32_t count = 0;
270 uint32_t width = 0;
271 uint32_t lights = 0;
272 };
273
274 317 std::vector<PrototypeEntry> entries;
275 317 std::vector<PrototypePiece> pieces;
276 317 std::vector<InstancerBlock> blocks;
277 317 std::vector<uint32_t> ranges;
278 317 std::vector<int64_t> hidden;
279 317 std::map<uint32_t, Placed> placed;
280 317 std::set<uint32_t> building;
281 317 std::map<uint32_t, const Instancer*> byPrim;
282
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 317 times.
 339 for (const Instancer& instancer : m_instancers) {
283
1/1
✓ Branch 1 taken 22 times.
 22 byPrim[instancer.prim] = &instancer;
284 }
285
1/1
✓ Branch 1 taken 317 times.
 317 m_blockItems.assign(m_worlds + 1, {});
286
1/1
✓ Branch 1 taken 317 times.
 317 m_lightCopies.assign(m_worlds + 1, {});
287 317 m_instancerPrims.clear();
288 317 m_watch.clear();
289
290
1/1
✓ Branch 1 taken 317 times.
 317 std::function<std::optional<InstancerBlock>(const Instancer&, uint32_t)> describe;
291 339 const auto place = [&](uint32_t root, uint32_t depth) {
292
2/2
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 21 times.
 22 if (const auto found = placed.find(root); found != placed.end()) {
293 1 return found->second;
294 }
295
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 21 times.
 21 if (building.contains(root)) {
296 ✗ return Placed{};
297 }
298 21 building.insert(root);
299 21 std::vector<PrototypePiece> local;
300 21 uint32_t start = 0;
301 21 uint32_t lightStart = 0;
302
3/3
✓ Branch 1 taken 21 times.
✓ Branch 3 taken 30 times.
✓ Branch 4 taken 21 times.
 51 for (const uint32_t drawable : m_drawables[root]) {
303
3/3
✓ Branch 1 taken 30 times.
✓ Branch 3 taken 25 times.
✓ Branch 4 taken 5 times.
 30 const uint32_t light = IsEmitter(uint32_t(prims.Record(drawable).schema)) ? 1 : 0;
304
1/1
✓ Branch 1 taken 30 times.
 30 local.push_back({.prim = drawable,
305 .start = start,
306 .slots = 1,
307 .nested = kNoNested,
308 .lightStart = lightStart,
309 .lights = light});
310 30 ++start;
311 30 lightStart += light;
312 }
313
3/3
✓ Branch 1 taken 21 times.
✓ Branch 3 taken 1 time.
✓ Branch 4 taken 21 times.
 22 for (const uint32_t nested : m_nested[root]) {
314 1 const auto instancer = byPrim.find(nested);
315
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 if (instancer == byPrim.end()) {
316 ✗ continue;
317 }
318
1/1
✓ Branch 1 taken 1 time.
 1 const std::optional<InstancerBlock> inner = describe(*instancer->second, depth + 1);
319
3/6
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 time.
✗ Branch 5 not taken.
 1 if (!inner || inner->count == 0 || inner->width == 0) {
320 ✗ continue;
321 }
322
1/1
✓ Branch 1 taken 1 time.
 1 local.push_back({.prim = nested,
323 .start = start,
324 1 .slots = inner->count * inner->width,
325
1/1
✓ Branch 1 taken 1 time.
 1 .nested = uint32_t(blocks.size()),
326 .lightStart = lightStart,
327
1/1
✓ Branch 1 taken 1 time.
 1 .lights = inner->count * inner->lightWidth});
328
1/1
✓ Branch 1 taken 1 time.
 1 blocks.push_back(*inner);
329 1 start += local.back().slots;
330 1 lightStart += local.back().lights;
331 }
332 21 building.erase(root);
333
1/1
✓ Branch 1 taken 21 times.
 21 const Placed result{.first = uint32_t(pieces.size()),
334 21 .count = uint32_t(local.size()),
335 .width = start,
336 21 .lights = lightStart};
337
1/1
✓ Branch 1 taken 21 times.
 21 pieces.insert(pieces.end(), local.begin(), local.end());
338
1/1
✓ Branch 1 taken 21 times.
 21 placed[root] = result;
339 21 return result;
340 21 };
341 ✗ describe = [&](const Instancer& instancer, uint32_t depth) -> std::optional<InstancerBlock> {
342
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
 22 if (depth >= kMostNesting) {
343 ✗ return std::nullopt;
344 }
345 22 std::vector<PrototypeEntry> mine;
346 22 uint32_t width = 0;
347 22 uint32_t lightWidth = 0;
348
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 22 times.
 44 for (const uint32_t root : instancer.roots) {
349 22 PrototypeEntry entry{
350 .first = 0, .count = 0, .parent = kNoPrim, .width = 0, .lights = 0};
351
1/2
✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
 22 if (root != kNoPrim) {
352
1/1
✓ Branch 1 taken 22 times.
 22 const Placed pieced = place(root, depth);
353 22 entry.first = pieced.first;
354 22 entry.count = pieced.count;
355 22 entry.width = pieced.width;
356 22 entry.lights = pieced.lights;
357
3/3
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 1 time.
✓ Branch 3 taken 21 times.
 22 entry.parent = instancer.native ? root : prims.Record(root).parent;
358
3/4
✓ Branch 0 taken 39 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 17 times.
✓ Branch 3 taken 22 times.
 78 while (entry.parent != kNoPrim &&
359
2/2
✓ Branch 1 taken 39 times.
✓ Branch 4 taken 39 times.
 39 !IsA(prims.Record(entry.parent).schema, SchemaId::Xformable)) {
360
1/1
✓ Branch 1 taken 17 times.
 17 entry.parent = prims.Record(entry.parent).parent;
361 }
362 }
363
1/2
✓ Branch 0 taken 22 times.
✗ Branch 1 not taken.
 22 width = std::max(width, entry.width);
364
2/2
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 17 times.
 22 lightWidth = std::max(lightWidth, entry.lights);
365
1/1
✓ Branch 1 taken 22 times.
 22 mine.push_back(entry);
366 }
367
1/1
✓ Branch 1 taken 22 times.
 22 const uint32_t first = uint32_t(entries.size());
368
1/1
✓ Branch 1 taken 22 times.
 22 entries.insert(entries.end(), mine.begin(), mine.end());
369
1/1
✓ Branch 1 taken 22 times.
 22 const uint32_t range = uint32_t(ranges.size() / 2);
370
1/1
✓ Branch 1 taken 22 times.
 22 ranges.push_back(uint32_t(hidden.size()));
371
2/2
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 1 time.
 22 if (!instancer.native) {
372
1/1
✓ Branch 1 taken 21 times.
 21 const ArrayRef ids = store.Array(kPointInstancerInvisibleIds,
373
1/1
✓ Branch 1 taken 21 times.
 21 prims.Row(instancer.prim, SchemaId::PointInstancer));
374
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 19 times.
 21 if (ids.address != 0) {
375 2 std::vector<int64_t> sorted =
376
3/3
✓ Branch 1 taken 2 times.
✓ Branch 4 taken 2 times.
✓ Branch 7 taken 2 times.
 2 Items<int64_t>(store.Arrays(ColumnType::Int64Array).Read(ids));
377 2 std::ranges::sort(sorted);
378
1/1
✓ Branch 1 taken 2 times.
 2 hidden.insert(hidden.end(), sorted.begin(), sorted.end());
379 2 }
380 }
381
1/1
✓ Branch 1 taken 22 times.
 22 ranges.push_back(uint32_t(hidden.size()) - ranges.back());
382 22 return InstancerBlock{.instancer = instancer.prim,
383 .first = 0,
384 22 .count = instancer.count,
385 .width = width,
386 .prototypes = first,
387
2/2
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 1 time.
 22 .prototypeCount = uint32_t(instancer.roots.size()),
388
2/2
✓ Branch 0 taken 21 times.
✓ Branch 1 taken 1 time.
 22 .native = instancer.native ? 1u : 0u,
389 .hidden = range,
390 22 .lightWidth = lightWidth};
391
1/1
✓ Branch 1 taken 317 times.
 339 };
392
393
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 317 times.
 339 for (const Instancer& instancer : m_instancers) {
394
1/1
✓ Branch 1 taken 22 times.
 22 m_instancerPrims.insert(instancer.prim);
395
2/2
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 21 times.
 22 if (m_members.contains(instancer.prim)) {
396 3 continue;
397 }
398
1/1
✓ Branch 1 taken 21 times.
 21 const std::optional<InstancerBlock> block = describe(instancer, 0);
399
4/6
✓ Branch 0 taken 21 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 21 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 2 times.
✓ Branch 5 taken 19 times.
 21 if (!block || block->width == 0 || block->count == 0) {
400 2 continue;
401 }
402 19 std::set<uint32_t> worlds;
403
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 19 times.
 38 for (const uint32_t root : instancer.roots) {
404
1/2
✓ Branch 0 taken 19 times.
✗ Branch 1 not taken.
 19 if (root != kNoPrim) {
405
3/3
✓ Branch 1 taken 19 times.
✓ Branch 3 taken 27 times.
✓ Branch 4 taken 19 times.
 46 for (const uint32_t drawable : m_drawables[root]) {
406
2/2
✓ Branch 1 taken 27 times.
✓ Branch 4 taken 27 times.
 27 worlds.insert(prims.Record(drawable).world);
407 }
408 }
409 }
410 39 const auto put = [&](uint32_t tlas) {
411 20 std::vector<uint32_t>& items = m_blockItems[tlas];
412 20 InstancerBlock placedBlock = *block;
413 20 placedBlock.first = uint32_t(items.size());
414 20 const uint32_t index = uint32_t(blocks.size());
415 20 blocks.push_back(placedBlock);
416 20 items.insert(items.end(), size_t(block->count) * block->width, kInstancerSlot | index);
417
2/2
✓ Branch 0 taken 10 times.
✓ Branch 1 taken 20 times.
 30 for (uint32_t offset = 0; offset < block->count * block->lightWidth; ++offset) {
418 10 m_lightCopies[tlas].push_back({index, offset});
419 }
420
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 20 times.
 39 for (const uint32_t world : worlds) {
421
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 19 times.
 19 if (world != instancer.world) {
422 ✗ m_watch[world].insert(tlas);
423 }
424 }
425 39 };
426
1/2
✓ Branch 0 taken 19 times.
✗ Branch 1 not taken.
 19 if (instancer.world == kGlobalWorld) {
427
2/2
✓ Branch 0 taken 20 times.
✓ Branch 1 taken 19 times.
 39 for (uint32_t tlas = 0; tlas <= m_worlds; ++tlas) {
428
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 if (ShowsGlobals(tlas)) {
429
1/1
✓ Branch 1 taken 20 times.
 20 put(tlas);
430 }
431 }
432
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ } else if (instancer.world < m_worlds) {
433
0/1
✗ Branch 1 not taken.
 ✗ put(instancer.world);
434
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 ✗ if (const auto shown = m_shownIn.find(instancer.world); shown != m_shownIn.end()) {
435
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ for (const uint32_t tlas : shown->second) {
436
0/1
✗ Branch 1 not taken.
 ✗ put(tlas);
437 }
438 }
439 }
440 19 }
441
442 317 std::vector<std::byte> tables;
443 1585 const auto table = [&](auto values) {
444 1585 const uint64_t at = RoundUp(tables.size(), 16);
445 1585 tables.resize(at);
446 1585 const std::span<const std::byte> bytes = std::as_bytes(std::span(values));
447 1585 tables.insert(tables.end(), bytes.begin(), bytes.end());
448 1585 return at;
449 };
450
2/2
✓ Branch 1 taken 317 times.
✓ Branch 4 taken 317 times.
 317 const uint64_t blocksStart = table(blocks);
451
2/2
✓ Branch 1 taken 317 times.
✓ Branch 4 taken 317 times.
 317 const uint64_t entriesStart = table(entries);
452
2/2
✓ Branch 1 taken 317 times.
✓ Branch 4 taken 317 times.
 317 const uint64_t piecesStart = table(pieces);
453
2/2
✓ Branch 1 taken 317 times.
✓ Branch 4 taken 317 times.
 317 const uint64_t rangesStart = table(ranges);
454
2/2
✓ Branch 1 taken 317 times.
✓ Branch 4 taken 317 times.
 317 const uint64_t hiddenStart = table(hidden);
455
3/3
✓ Branch 0 taken 297 times.
✓ Branch 1 taken 20 times.
✓ Branch 3 taken 317 times.
 614 tables.resize(std::max<size_t>(RoundUp(tables.size(), 16), 16));
456
1/1
✓ Branch 1 taken 317 times.
 317 m_instancing =
457
4/4
✓ Branch 1 taken 317 times.
✓ Branch 4 taken 317 times.
✓ Branch 6 taken 17 times.
✓ Branch 7 taken 300 times.
 334 store.GetDevice().CreateBuffer({.label = "instancers"}, std::span<const std::byte>(tables));
458 317 const uint64_t instancing = m_instancing->Address();
459 317 m_header.blocks.address = instancing + blocksStart;
460 317 m_header.prototypes.address = instancing + entriesStart;
461 317 m_header.pieces.address = instancing + piecesStart;
462 317 m_header.hiddenRanges.address = instancing + rangesStart;
463 317 m_header.hiddenIds = instancing + hiddenStart;
464 634 }
465
466 317 void InstanceTable::WriteLayoutHeader() {
467
1/2
✓ Branch 0 taken 317 times.
✗ Branch 1 not taken.
 317 m_header.handles.address = m_tlas->Handles().Address();
468
1/2
✓ Branch 0 taken 317 times.
✗ Branch 1 not taken.
 317 m_header.shapes.address = m_shapes.Address();
469
1/2
✓ Branch 0 taken 317 times.
✗ Branch 1 not taken.
 317 m_header.hits.address = m_hits.Address();
470
1/2
✓ Branch 0 taken 317 times.
✗ Branch 1 not taken.
 317 m_header.records = m_records.Address();
471 317 m_header.worldCount = m_worlds;
472 317 m_header.displayColorName = m_structure.GetTokens().Intern("displayColor").id;
473 317 m_structure.GetColumnStore().GetDevice().GetQueue().Write(m_layout->Slice(0, sizeof(m_header)),
474 317 &m_header);
475 317 }
476
477 1065 device::BufferRange InstanceTable::LayoutWords(uint64_t address, uint32_t first,
478 uint32_t count) const {
479 1065 return m_layout->Slice(address - m_layout->Address() + uint64_t(first) * sizeof(uint32_t),
480 1065 uint64_t(count) * sizeof(uint32_t));
481 }
482
483 187 void InstanceTable::WriteRegions(const std::set<uint32_t>& tlases) {
484
2/2
✓ Branch 0 taken 136 times.
✓ Branch 1 taken 51 times.
 187 if (tlases.empty()) {
485 return;
486 }
487 136 device::Queue& queue = m_structure.GetColumnStore().GetDevice().GetQueue();
488
2/2
✓ Branch 0 taken 139 times.
✓ Branch 1 taken 136 times.
 275 for (const uint32_t tlas : tlases) {
489 139 const uint32_t start = m_regions.Starts()[tlas];
490 139 queue.Write(LayoutWords(m_header.instancePrims.address, start, m_regions.Capacity(tlas)),
491 278 m_regions.Items().data() + start);
492 }
493 136 queue.Write(LayoutWords(m_header.instanceCounts.address, 0, m_regions.Size()),
494 272 m_regions.Counts().data());
495 }
496
497 6 bool InstanceTable::Restack() {
498 6 BuildInstancing();
499 6 std::set<uint32_t> tlases;
500
2/2
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 6 times.
 12 for (uint32_t tlas = 0; tlas < m_regions.Size(); ++tlas) {
501
3/4
✓ Branch 1 taken 6 times.
✓ Branch 4 taken 6 times.
✗ Branch 7 not taken.
✓ Branch 8 taken 6 times.
 6 if (!m_regions.Replace(tlas, RegionItems(tlas))) {
502 ✗ return false;
503 }
504
1/1
✓ Branch 1 taken 6 times.
 6 tlases.insert(tlas);
505 }
506
1/1
✓ Branch 1 taken 6 times.
 6 WriteRegions(tlases);
507
1/1
✓ Branch 1 taken 6 times.
 6 WriteLayoutHeader();
508 return true;
509 6 }
510
511 191 std::optional<std::set<uint32_t>> InstanceTable::Apply(const ChangeSet& changes) {
512
4/5
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 188 times.
✓ Branch 3 taken 3 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 3 times.
 191 CHRONO_BATCH_RENDER_ZONE("Instances/Apply");
513
2/2
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 186 times.
 191 const Prims& prims = m_structure.GetPrims();
514
2/2
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 186 times.
 191 if (prims.WorldCount() != m_worlds) {
515 5 return std::nullopt;
516 }
517 186 std::set<uint32_t> touched;
518 5394 const auto touch = [&](uint32_t world) {
519
2/2
✓ Branch 0 taken 5204 times.
✓ Branch 1 taken 4 times.
 5208 if (world == kGlobalWorld) {
520
2/2
✓ Branch 0 taken 5207 times.
✓ Branch 1 taken 5204 times.
 10411 for (uint32_t tlas = 0; tlas <= m_worlds; ++tlas) {
521 5207 touched.insert(tlas);
522 }
523 } else {
524 4 touched.insert(world);
525
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 4 times.
 4 if (const auto shown = m_shownIn.find(world); shown != m_shownIn.end()) {
526 ✗ touched.insert(shown->second.begin(), shown->second.end());
527 }
528 }
529 5394 };
530
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 186 times.
 197 for (const StructureEntry& entry : changes.apis) {
531
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (entry.api == SchemaId::PrototypeAPI) {
532 ✗ return std::nullopt;
533 }
534 }
535
2/2
✓ Branch 0 taken 142 times.
✓ Branch 1 taken 185 times.
 327 for (const StructureEntry& entry : changes.removed) {
536
2/4
✓ Branch 2 taken 142 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 142 times.
✗ Branch 5 not taken.
 426 if (m_origins.contains(entry.prim) || m_instancerPrims.contains(entry.prim) ||
537
3/4
✓ Branch 0 taken 142 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 1 time.
✓ Branch 4 taken 141 times.
 426 m_members.contains(entry.prim) ||
538
2/2
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 141 times.
 142 std::ranges::find(m_products, entry.prim) != m_products.end()) {
539 1 return std::nullopt;
540 }
541
2/2
✓ Branch 0 taken 139 times.
✓ Branch 1 taken 2 times.
 141 std::vector<uint32_t>& list = entry.world == kGlobalWorld ? m_globals : m_own[entry.world];
542
2/2
✓ Branch 1 taken 95 times.
✓ Branch 2 taken 46 times.
 141 if (std::erase(list, entry.prim) > 0) {
543
1/1
✓ Branch 1 taken 95 times.
 95 touch(entry.world);
544 }
545 }
546
2/2
✓ Branch 0 taken 5241 times.
✓ Branch 1 taken 184 times.
 5425 for (const StructureEntry& entry : changes.added) {
547
6/6
✓ Branch 1 taken 5241 times.
✓ Branch 3 taken 5225 times.
✓ Branch 4 taken 16 times.
✓ Branch 6 taken 5225 times.
✓ Branch 8 taken 5215 times.
✓ Branch 9 taken 10 times.
 5241 if (!prims.Alive(entry.prim) || prims.Generation(entry.prim) != entry.generation) {
548 26 continue;
549 }
550
1/1
✓ Branch 1 taken 5215 times.
 5215 const SchemaId schema = prims.Record(entry.prim).schema;
551
1/1
✓ Branch 1 taken 5215 times.
 5215 if (IsA(schema, SchemaId::World) || schema == SchemaId::RenderProduct ||
552
3/4
✓ Branch 0 taken 5214 times.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 5214 times.
✗ Branch 3 not taken.
 10429 schema == SchemaId::PointInstancer ||
553
4/6
✓ Branch 0 taken 5215 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 5214 times.
✓ Branch 6 taken 5214 times.
✓ Branch 8 taken 5214 times.
✗ Branch 9 not taken.
 15643 prims.HasApi(entry.prim, SchemaId::PrototypeAPI) || Member(entry.prim)) {
554 1 return std::nullopt;
555 }
556
3/3
✓ Branch 1 taken 5214 times.
✓ Branch 3 taken 5113 times.
✓ Branch 4 taken 101 times.
 5214 if (Drawable(schema)) {
557
3/3
✓ Branch 0 taken 5111 times.
✓ Branch 1 taken 2 times.
✓ Branch 3 taken 5113 times.
 5113 (entry.world == kGlobalWorld ? m_globals : m_own[entry.world]).push_back(entry.prim);
558
1/1
✓ Branch 1 taken 5113 times.
 5113 touch(entry.world);
559 }
560 }
561
2/2
✓ Branch 0 taken 136 times.
✓ Branch 1 taken 181 times.
 317 for (const uint32_t tlas : touched) {
562
4/4
✓ Branch 1 taken 136 times.
✓ Branch 4 taken 136 times.
✓ Branch 7 taken 3 times.
✓ Branch 8 taken 133 times.
 136 if (!m_regions.Replace(tlas, RegionItems(tlas))) {
563 3 return std::nullopt;
564 }
565 }
566
567
1/1
✓ Branch 1 taken 181 times.
 181 WriteRegions(touched);
568 181 return touched;
569 191 }
570
571 }
572