GCC Code Coverage Report

chrono_batch_render/
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
450 / 481
93.6%
Branches:
284 / 377
75.3%
Functions:
56 / 57
98.2%
stage/columns/column_store.cpp
Line Branch Exec Source
1 // SPDX-License-Identifier: BSD-3-Clause
2 #include "chrono_batch_render/stage/columns/column_store.h"
3
4 #include <algorithm>
5 #include <cstring>
6 #include <span>
7
8 #include <fmt/format.h>
9
10 #include "chrono_batch_render/device/commands.h"
11 #include "chrono_batch_render/device/queue.h"
12 #include "chrono_batch_render/stage/columns/kernels/cells.h"
13 #include "chrono_batch_render/stage/kernels.h"
14 #include "chrono_batch_render/util/error.h"
15 #include "chrono_batch_render/util/profiler.h"
16 #include "chrono_batch_render/util/span.h"
17
18 namespace chrono::batch_render {
19
20 namespace {
21
22 constexpr uint32_t kNoPrimFill = 0xFFFFFFFF;
23 constexpr uint64_t kLargestStage = 64 * 1024;
24
25 85522 std::vector<std::byte> Repeat(std::span<const std::byte> cell, uint32_t count) {
26 85522 std::vector<std::byte> bytes;
27
1/1
✓ Branch 1 taken 85522 times.
 85522 bytes.reserve(cell.size() * count);
28
2/2
✓ Branch 0 taken 301043 times.
✓ Branch 1 taken 85522 times.
 386565 for (uint32_t i = 0; i < count; ++i) {
29
1/1
✓ Branch 1 taken 301043 times.
 301043 bytes.insert(bytes.end(), cell.begin(), cell.end());
30 }
31 85522 return bytes;
32 ✗ }
33
34 56338 std::vector<TableColumn> SchemaColumns(SchemaId schema) {
35 56338 std::vector<TableColumn> columns;
36
2/2
✓ Branch 0 taken 568 times.
✓ Branch 1 taken 55770 times.
 56338 const uint32_t count = Info(schema).columnCount;
37
2/2
✓ Branch 0 taken 186008 times.
✓ Branch 1 taken 56338 times.
 242346 for (uint32_t column = 0; column < count; ++column) {
38
2/2
✓ Branch 1 taken 186008 times.
✓ Branch 4 taken 186008 times.
 186008 columns.push_back({ElementBytes(Columns(schema)[column].type), 0});
39 }
40
1/1
✓ Branch 1 taken 56338 times.
 56338 columns.push_back({sizeof(uint32_t), kNoPrimFill});
41 56338 return columns;
42 ✗ }
43
44 }
45
46 507 ColumnStore::ColumnStore(device::Device& device)
47
1/1
✓ Branch 1 taken 507 times.
 507 : m_device(device), m_heap(device, "scene"),
48
4/4
✓ Branch 1 taken 507 times.
✓ Branch 4 taken 507 times.
✓ Branch 7 taken 507 times.
✓ Branch 11 taken 507 times.
 1014 m_prims(m_heap, "prims",
49 {{4, 0}, {4, kNoPrimFill}, {4, kNoPrimFill}, {4, 0}, {4, kNoPrimFill}, {8, 0}}),
50
4/4
✓ Branch 1 taken 507 times.
✓ Branch 4 taken 507 times.
✓ Branch 7 taken 507 times.
✓ Branch 11 taken 507 times.
 1014 m_slots(m_heap, "prim slots", {{4, kNoPrimFill}}),
51
6/6
✓ Branch 2 taken 507 times.
✓ Branch 5 taken 507 times.
✓ Branch 8 taken 507 times.
✓ Branch 11 taken 507 times.
✓ Branch 15 taken 507 times.
✓ Branch 18 taken 507 times.
 2028 m_worlds(m_heap, "world stamps", {{sizeof(uint64_t), 0}}) {
52
1/1
✓ Branch 1 taken 507 times.
 507 m_entries.reserve(uint32_t(SchemaId::Count));
53
1/1
✓ Branch 1 taken 507 times.
 507 Lookup(SchemaId(uint32_t(SchemaId::Count) - 1));
54
1/1
✓ Branch 1 taken 507 times.
 507 m_worlds.Allocate(1);
55 507 }
56
57 1647277 const ColumnStore::Entry& ColumnStore::Lookup(SchemaId schema) const {
58
1/2
✓ Branch 1 taken 1647277 times.
✗ Branch 2 not taken.
 1647277 if (uint32_t(schema) >= SchemaCount()) {
59
0/1
✗ Branch 2 not taken.
 ✗ throw Error("No data for schema {}", uint32_t(schema));
60 }
61
2/2
✓ Branch 0 taken 56338 times.
✓ Branch 1 taken 1647277 times.
 1703615 while (m_entries.size() <= uint32_t(schema)) {
62 56338 const SchemaId added = SchemaId(uint32_t(m_entries.size()));
63 56338 Entry entry;
64 56338 entry.table = std::make_unique<ColumnTable>(
65
3/3
✓ Branch 1 taken 56338 times.
✓ Branch 4 taken 56338 times.
✓ Branch 7 taken 56338 times.
 225352 m_heap, fmt::format("{} data", Info(added).name), SchemaColumns(added));
66
3/3
✓ Branch 0 taken 568 times.
✓ Branch 1 taken 55770 times.
✓ Branch 3 taken 56338 times.
 112676 entry.stamps.assign(Info(added).columnCount, kNoStamp);
67
3/3
✓ Branch 0 taken 568 times.
✓ Branch 1 taken 55770 times.
✓ Branch 3 taken 56338 times.
 112676 entry.writes.assign(Info(added).columnCount, 0);
68
1/1
✓ Branch 1 taken 56338 times.
 56338 m_entries.push_back(std::move(entry));
69 56338 }
70 1647277 return m_entries[uint32_t(schema)];
71 }
72
73 410334 ColumnTable& ColumnStore::Schema(SchemaId schema) {
74 410334 return *Lookup(schema).table;
75 }
76
77 ✗ const ColumnTable& ColumnStore::Schema(SchemaId schema) const {
78 ✗ return *Lookup(schema).table;
79 }
80
81 40 void ColumnStore::Pin(SchemaId schema) {
82 40 Lookup(schema);
83 40 ++m_entries[uint32_t(schema)].pins;
84 40 }
85
86 39 void ColumnStore::Unpin(SchemaId schema) {
87
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 39 times.
 39 if (!Pinned(schema)) {
88
0/3
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
 ✗ throw Error("{} data isn't pinned", Info(schema).name);
89 }
90 39 --m_entries[uint32_t(schema)].pins;
91 39 }
92
93 507 ColumnStore::~ColumnStore() {
94
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 506 times.
 507 if (!m_attachments.empty()) {
95 1 try {
96
2/2
✓ Branch 1 taken 1 time.
✓ Branch 4 taken 1 time.
 1 m_device.GetQueue().Mark().Wait();
97 ✗ } catch (...) {
98 ✗ }
99 }
100 1337 }
101
102 namespace {
103
104 20 bool SameColumn(ColumnRef a, ColumnRef b) {
105
2/4
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 20 times.
 20 return a.schema == b.schema && a.index == b.index;
106 }
107
108 }
109
110 8 void ColumnStore::Attach(Attachment attachment) {
111
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 8 times.
 8 if (Attached(attachment.column, attachment.rows)) {
112 ✗ throw Error("{}'s {} is already attached for some of these prims",
113
0/3
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
 ✗ Info(attachment.column.schema).name, attachment.column.column->field);
114 }
115 8 attachment.sorted = attachment.rows;
116 8 std::ranges::sort(attachment.sorted);
117 8 Pin(attachment.column.schema);
118 8 m_attachments.push_back(std::move(attachment));
119 8 }
120
121 9 const Attachment* ColumnStore::FindAttachment(ColumnRef column,
122 std::span<const uint32_t> rows) const {
123
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 for (const Attachment& attachment : m_attachments) {
124
2/4
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 9 times.
✗ Branch 3 not taken.
 9 if (SameColumn(attachment.column, column) && std::ranges::equal(attachment.rows, rows)) {
125 9 return &attachment;
126 }
127 }
128 ✗ return nullptr;
129 }
130
131 142 bool ColumnStore::Attached(ColumnRef column, std::span<const uint32_t> rows) const {
132
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 138 times.
 142 for (const Attachment& attachment : m_attachments) {
133
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 4 times.
 4 if (!SameColumn(attachment.column, column)) {
134 ✗ continue;
135 }
136
1/2
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
 4 for (const uint32_t row : rows) {
137
2/4
✓ Branch 0 taken 4 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 4 times.
✗ Branch 4 not taken.
 4 if (row != kNoRow && std::ranges::binary_search(attachment.sorted, row)) {
138 4 return true;
139 }
140 }
141 }
142 138 return false;
143 }
144
145 8 Attachment ColumnStore::Detach(ColumnRef column, std::span<const uint32_t> rows) {
146 8 const auto found = std::ranges::find_if(m_attachments, [&](const Attachment& attachment) {
147
2/4
✓ Branch 0 taken 7 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 7 times.
 7 return SameColumn(attachment.column, column) && std::ranges::equal(attachment.rows, rows);
148 });
149
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 7 times.
 8 if (found == m_attachments.end()) {
150 3 throw Error("{}'s {} isn't attached for these prims", Info(column.schema).name,
151
2/3
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✓ Branch 4 taken 1 time.
 2 column.column->field);
152 }
153 7 Attachment attachment = std::move(*found);
154 7 m_attachments.erase(found);
155
1/1
✓ Branch 1 taken 7 times.
 7 Unpin(column.schema);
156 7 return attachment;
157 ✗ }
158
159 2445 bool ColumnStore::Pinned(SchemaId schema) const {
160 2445 return Lookup(schema).pins > 0;
161 }
162
163 7161 ArrayPool& ColumnStore::Arrays(ColumnType type) {
164 7161 const uint32_t item = ItemBytes(type);
165
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7161 times.
 7161 if (item == 0) {
166
0/1
✗ Branch 2 not taken.
 ✗ throw Error("Column type {} holds no arrays", uint32_t(type));
167 }
168 7161 std::unique_ptr<ArrayPool>& arrays = m_arrayHeaps[item];
169
2/2
✓ Branch 0 taken 560 times.
✓ Branch 1 taken 6601 times.
 7161 if (!arrays) {
170
1/1
✓ Branch 2 taken 560 times.
 1120 arrays = std::make_unique<ArrayPool>(m_heap, fmt::format("{}-byte arrays", item), item);
171 }
172 7161 return *arrays;
173 }
174
175 194720 uint32_t ColumnStore::PrimsColumn(SchemaId schema) const {
176 194720 Lookup(schema);
177
2/2
✓ Branch 0 taken 27 times.
✓ Branch 1 taken 194693 times.
 194720 return Info(schema).columnCount;
178 }
179
180 679387 uint32_t ColumnStore::StampColumn(ColumnRef column) const {
181 679387 const std::vector<uint32_t>& stamps = Lookup(column.schema).stamps;
182
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 679387 times.
 679387 if (column.index >= stamps.size()) {
183
0/3
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
 ✗ throw Error("{} has no column {}", Info(column.schema).name, column.index);
184 }
185 679387 return stamps[column.index];
186 }
187
188 9764 bool ColumnStore::Tracked(ColumnRef column) const {
189 9764 return StampColumn(column) != kNoStamp;
190 }
191
192 9759 void ColumnStore::TrackRows(ColumnRef column) {
193
2/2
✓ Branch 1 taken 4071 times.
✓ Branch 2 taken 5688 times.
 9759 if (Tracked(column)) {
194 4071 return;
195 }
196 5688 Entry& entry = m_entries[uint32_t(column.schema)];
197 5688 const uint32_t stamps = entry.table->AddColumn({sizeof(uint64_t), 0});
198 5688 entry.stamps[column.index] = stamps;
199 5688 const Ordinal published = m_stamp - 1;
200 5688 FillWords(*entry.table, stamps, 0, entry.table->Rows(),
201 std::as_bytes(std::span(&published, 1)));
202
1/1
✓ Branch 1 taken 5688 times.
 11376 entry.table->Flush();
203 }
204
205 150075 void ColumnStore::Touch(ColumnRef column) {
206 150075 StampColumn(column);
207 150075 m_entries[uint32_t(column.schema)].writes[column.index] = ++m_writes;
208 150075 }
209
210 359844 uint64_t ColumnStore::LastWrite(ColumnRef column) const {
211 359844 StampColumn(column);
212 359844 return Lookup(column.schema).writes[column.index];
213 }
214
215 70280 StampTarget ColumnStore::Stamps(ColumnRef column) {
216 70280 const uint32_t stamps = StampColumn(column);
217 70280 const ColumnTable& table = Schema(column.schema);
218 70280 StampTarget target{};
219
2/2
✓ Branch 0 taken 70245 times.
✓ Branch 1 taken 35 times.
 70280 target.stamps.address = stamps == kNoStamp ? 0 : table.Pages(stamps);
220 70280 target.tablePrims.address = table.Pages(PrimsColumn(column.schema));
221 70280 target.scene.address = DeviceData();
222 70280 target.stamp = m_stamp;
223 70280 return target;
224 }
225
226 81670 void ColumnStore::StampWorld(uint32_t world) {
227
2/2
✓ Branch 0 taken 441 times.
✓ Branch 1 taken 81229 times.
 81670 const uint32_t slot = WorldSlot(world);
228
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 81670 times.
 81670 if (slot >= m_worlds.Rows()) {
229
0/1
✗ Branch 2 not taken.
 ✗ throw Error("World {} of a scene of {}", world, m_worldCount);
230 }
231 81670 m_worlds.Stage(0, slot, std::as_bytes(std::span(&m_stamp, 1)));
232 81670 m_worldStamps[slot] = m_stamp;
233 81670 }
234
235 12006 Ordinal ColumnStore::WorldStamp(uint32_t world) const {
236
2/2
✓ Branch 0 taken 1532 times.
✓ Branch 1 taken 10474 times.
 12006 const uint32_t slot = WorldSlot(world);
237
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 12006 times.
 12006 if (slot >= m_worldStamps.size()) {
238
0/1
✗ Branch 2 not taken.
 ✗ throw Error("World {} of a scene of {}", world, m_worldCount);
239 }
240 12006 return m_worldStamps[slot];
241 }
242
243 37843 device::Program& ColumnStore::Kernels() {
244
2/2
✓ Branch 0 taken 428 times.
✓ Branch 1 taken 37415 times.
 37843 if (!m_program) {
245
2/2
✓ Branch 1 taken 428 times.
✓ Branch 4 taken 428 times.
 1284 m_program = m_device.CreateProgram(
246 {.modules = {"chrono_batch_render.stage.columns.kernels.cells",
247 "chrono_batch_render.stage.columns.kernels.changes"},
248 .library = &kStageKernels,
249 .entries = {"CopyWords", "ScatterCells", "StampRows", "GatherCells", "FillCells",
250 "CloneRows", "WritePoses", "ReadPoses", "CompactChanged", "GatherRows"},
251
2/3
✓ Branch 1 taken 428 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 428 times.
 856 .label = "scene store"});
252 }
253 37843 return *m_program;
254 856 }
255
256 274 device::Download ColumnStore::ReadRowsAsync(const Ref<device::Buffer>& source, uint32_t stride,
257 std::span<const uint32_t> rows) {
258
3/4
✓ Branch 0 taken 273 times.
✓ Branch 1 taken 1 time.
✗ Branch 2 not taken.
✓ Branch 3 taken 273 times.
 274 if (rows.empty() || !source) {
259 1 return {};
260 }
261 273 const uint64_t bytes = uint64_t(rows.size()) * stride;
262
1/1
✓ Branch 1 taken 273 times.
 273 const Ref<device::Buffer> out = m_device.CreateBuffer({.bytes = bytes, .label = "read rows"});
263
1/1
✓ Branch 1 taken 273 times.
 273 const Ref<device::Buffer> list = m_device.CreateBuffer({.label = "read row list"}, rows);
264 273 GatherRowsParams params{};
265 273 params.source.address = source->Address();
266 273 params.rows.address = list->Address();
267 273 params.out.address = out->Address();
268 273 params.words = stride / uint32_t(sizeof(uint32_t));
269
1/1
✓ Branch 1 taken 273 times.
 273 params.count = uint32_t(rows.size());
270
1/1
✓ Branch 1 taken 273 times.
 273 Dispatch("GatherRows", params, params.count);
271
2/2
✓ Branch 1 taken 273 times.
✓ Branch 4 taken 273 times.
 273 return m_device.GetQueue().ReadAsync(out->Slice(0, bytes));
272 1365 }
273
274 91214 void ColumnStore::FillWords(ColumnTable& table, uint32_t column, uint32_t first, uint32_t count,
275 std::span<const std::byte> cell) {
276
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 91214 times.
 91214 if (cell.size() != table.CellBytes(column)) {
277
0/2
✗ Branch 1 not taken.
✗ Branch 4 not taken.
 ✗ throw Error("{} bytes for a cell of {} bytes in {}", cell.size(), table.CellBytes(column),
278
0/2
✗ Branch 2 not taken.
✗ Branch 5 not taken.
 ✗ table.Label());
279 }
280
2/2
✓ Branch 0 taken 85528 times.
✓ Branch 1 taken 5686 times.
 91214 if (count == 0) {
281 91207 return;
282 }
283
2/2
✓ Branch 0 taken 85521 times.
✓ Branch 1 taken 7 times.
 85528 if (cell.size() * count <= kLargestStage) {
284
1/1
✓ Branch 2 taken 85521 times.
 85521 table.Stage(column, first, Repeat(cell, count));
285 85521 return;
286 }
287
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 if (cell.size() > kMaxFillWords * sizeof(uint32_t)) {
288
0/1
✗ Branch 2 not taken.
 ✗ throw Error("Cells of {} bytes are too wide to fill", cell.size());
289 }
290
1/1
✓ Branch 1 taken 7 times.
 7 table.Flush();
291 7 FillCellsParams params{};
292 7 params.column.address = table.Pages(column);
293 7 params.first = first;
294 7 params.count = count;
295 7 params.words = uint32_t(cell.size() / sizeof(uint32_t));
296 7 std::memcpy(params.cell, cell.data(), cell.size());
297 7 Dispatch("FillCells", params, uint64_t(count) * params.words);
298 }
299
300 79795 void ColumnStore::StampRows(ColumnRef column, uint32_t first, uint32_t count) {
301 79795 const uint32_t stamps = StampColumn(column);
302
2/2
✓ Branch 0 taken 10265 times.
✓ Branch 1 taken 69530 times.
 79795 if (stamps != kNoStamp) {
303 10265 FillWords(Schema(column.schema), stamps, first, count,
304 std::as_bytes(std::span(&m_stamp, 1)));
305 }
306 79795 }
307
308 3329 void ColumnStore::StageLive(ColumnRef column, uint32_t first, std::span<const std::byte> cells) {
309
2/2
✓ Branch 0 taken 2648 times.
✓ Branch 1 taken 681 times.
 3329 CheckStoreLock(m_lock, "Writing a scene's columns");
310 3329 ColumnTable& table = Schema(column.schema);
311 3329 table.Stage(column.index, first, cells);
312 3329 Touch(column);
313 3329 StampRows(column, first, uint32_t(cells.size() / table.CellBytes(column.index)));
314 3329 }
315
316 78585 bool ColumnStore::OverridesBetween(ColumnRef column, uint32_t first, uint32_t count) const {
317
3/4
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 78583 times.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 78585 if (m_overrides.empty() || count == 0) {
318 return false;
319 }
320 2 const auto found = m_overrides.lower_bound(ArrayKey(column, first));
321
2/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 2 times.
 2 return found != m_overrides.end() && found->first <= ArrayKey(column, first + count - 1);
322 }
323
324 3323 void ColumnStore::Stage(ColumnRef column, uint32_t first, std::span<const std::byte> cells) {
325 3323 const uint32_t cell = Schema(column.schema).CellBytes(column.index);
326 3323 const auto count = uint32_t(cells.size() / cell);
327
2/2
✓ Branch 1 taken 3322 times.
✓ Branch 2 taken 1 time.
 3323 if (!OverridesBetween(column, first, count)) {
328 3322 StageLive(column, first, cells);
329 3322 return;
330 }
331 1 std::vector<std::byte> live(cells.begin(), cells.end());
332 1 for (auto at = m_overrides.lower_bound(ArrayKey(column, first));
333
3/4
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 2 at != m_overrides.end() && at->first <= ArrayKey(column, first + count - 1); ++at) {
334
1/1
✓ Branch 1 taken 1 time.
 1 const size_t offset = size_t(uint32_t(at->first) - first) * cell;
335
1/1
✓ Branch 1 taken 1 time.
 1 at->second.authored.assign(cells.begin() + offset, cells.begin() + offset + cell);
336 1 std::ranges::copy(at->second.layers.back().value, live.begin() + offset);
337 }
338
1/1
✓ Branch 1 taken 1 time.
 1 StageLive(column, first, live);
339 1 }
340
341 75262 void ColumnStore::Fill(ColumnRef column, uint32_t first, uint32_t count,
342 std::span<const std::byte> cell) {
343
2/2
✓ Branch 0 taken 71885 times.
✓ Branch 1 taken 3377 times.
 75262 CheckStoreLock(m_lock, "Writing a scene's columns");
344
2/2
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 75261 times.
 75262 if (OverridesBetween(column, first, count)) {
345
1/1
✓ Branch 2 taken 1 time.
 1 Stage(column, first, Repeat(cell, count));
346 1 return;
347 }
348 75261 FillWords(Schema(column.schema), column.index, first, count, cell);
349 75261 Touch(column);
350 75261 StampRows(column, first, count);
351 }
352
353 134844 void ColumnStore::LandAuthored(bool wait) const {
354 134844 std::erase_if(m_authoredReads, [&](AuthoredRead& pending) {
355
2/4
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 3 taken 1 time.
✗ Branch 4 not taken.
 1 if (!wait && !pending.read.Ready()) {
356 return false;
357 }
358 1 pending.read.Wait();
359 1 const auto found = m_overrides.find(pending.key);
360 1 const Span<const std::byte> bytes = pending.read.Bytes();
361
2/4
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1 if (found != m_overrides.end() && bytes.size() == found->second.authored.size() &&
362
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 !std::ranges::equal(bytes, found->second.layers.back().value)) {
363 1 found->second.authored.assign(bytes.begin(), bytes.end());
364 }
365 return true;
366 });
367 134844 }
368
369 3 void ColumnStore::Override(ColumnRef column, uint32_t row, uint64_t owner,
370 std::span<const std::byte> authored, std::span<const std::byte> value) {
371 3 LandAuthored(true);
372 3 const uint32_t cell = Schema(column.schema).CellBytes(column.index);
373
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3 times.
 3 if (value.size() != cell) {
374
0/1
✗ Branch 1 not taken.
 ✗ throw Error("{} bytes override {}, whose cells are {}", value.size(), column.column->field,
375
0/1
✗ Branch 2 not taken.
 ✗ cell);
376 }
377 3 OverrideCell& entry = m_overrides[ArrayKey(column, row)];
378
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 1 time.
 3 if (entry.layers.empty()) {
379 2 entry.authored.assign(authored.begin(), authored.end());
380 }
381 3 const auto layer = std::ranges::find(entry.layers, owner, &OverrideLayer::owner);
382
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 3 times.
 3 if (layer != entry.layers.end()) {
383 ✗ layer->value.assign(value.begin(), value.end());
384 } else {
385 3 entry.layers.push_back({owner, {value.begin(), value.end()}});
386 3 std::ranges::sort(entry.layers, {}, &OverrideLayer::owner);
387 }
388 3 StageLive(column, row, entry.layers.back().value);
389 9 }
390
391 2 void ColumnStore::ClearOverride(ColumnRef column, uint32_t row, uint64_t owner) {
392 2 LandAuthored(true);
393 2 const auto found = m_overrides.find(ArrayKey(column, row));
394
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 if (found == m_overrides.end()) {
395 ✗ return;
396 }
397 2 std::erase_if(found->second.layers,
398
2/16
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✓ Branch 11 taken 1 time.
✓ Branch 12 taken 2 times.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
 3 [&](const OverrideLayer& layer) { return layer.owner == owner; });
399
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 1 time.
 2 if (found->second.layers.empty()) {
400 1 const std::vector<std::byte> authored = std::move(found->second.authored);
401 1 m_overrides.erase(found);
402
1/1
✓ Branch 1 taken 1 time.
 1 StageLive(column, row, authored);
403 1 } else {
404 1 StageLive(column, row, found->second.layers.back().value);
405 }
406 }
407
408 8 bool ColumnStore::Overridden(ColumnRef column, uint32_t row) const {
409 8 return m_overrides.contains(ArrayKey(column, row));
410 }
411
412 2 std::vector<std::byte> ColumnStore::Authored(ColumnRef column, uint32_t row) const {
413 2 LandAuthored(true);
414 2 const auto found = m_overrides.find(ArrayKey(column, row));
415
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 return found == m_overrides.end() ? std::vector<std::byte>() : found->second.authored;
416 }
417
418 70278 void ColumnStore::Reassert(ColumnRef column, std::span<const uint32_t> rows) {
419
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 70277 times.
 70278 if (m_overrides.empty()) {
420 return;
421 }
422 1 ColumnTable& table = Schema(column.schema);
423
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 1 time.
 2 for (const uint32_t row : rows) {
424 1 if (const auto found = m_overrides.find(ArrayKey(column, row));
425
1/2
✓ Branch 0 taken 1 time.
✗ Branch 1 not taken.
 1 found != m_overrides.end()) {
426 2 if (const std::optional<device::BufferRange> cell =
427
1/2
✓ Branch 1 taken 1 time.
✗ Branch 2 not taken.
 1 table.Contiguous(column.index, row, 1)) {
428 1 m_authoredReads.push_back(
429 1 {ArrayKey(column, row), m_device.GetQueue().ReadAsync(*cell)});
430 }
431 1 StageLive(column, row, found->second.layers.back().value);
432 }
433 }
434 1 }
435
436 1214 void ColumnStore::DropOverrides(SchemaId table, uint32_t row) {
437
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 1210 times.
 1214 if (m_overrides.empty()) {
438 return;
439 }
440 4 LandAuthored(true);
441
3/4
✗ Branch 1 not taken.
✓ Branch 2 taken 16 times.
✓ Branch 3 taken 12 times.
✓ Branch 4 taken 4 times.
 32 for (uint32_t index = 0; index < Info(table).columnCount; ++index) {
442 12 m_overrides.erase(ArrayKey({table, &Columns(table)[index], index}, row));
443 }
444 }
445
446 338 void ColumnStore::CloneRows(SchemaId table, std::span<const uint32_t> sources, uint32_t first,
447 uint32_t copies) {
448
2/4
✓ Branch 0 taken 338 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 338 times.
✗ Branch 3 not taken.
 338 if (sources.empty() || copies == 0) {
449 ✗ return;
450 }
451 338 Flush();
452 338 ColumnTable& rows = Schema(table);
453
1/1
✓ Branch 1 taken 338 times.
 338 const Ref<device::Buffer> sourceRows = m_device.CreateBuffer({.label = "cloned rows"}, sources);
454 338 const uint32_t count = uint32_t(sources.size());
455
1/1
✓ Branch 1 taken 338 times.
 338 const Span<const ColumnInfo> columns = Columns(table);
456
3/4
✗ Branch 0 not taken.
✓ Branch 1 taken 1543 times.
✓ Branch 2 taken 1205 times.
✓ Branch 3 taken 338 times.
 3086 for (uint32_t index = 0; index < Info(table).columnCount; ++index) {
457
1/1
✓ Branch 1 taken 1205 times.
 1205 const ColumnInfo& info = columns[index];
458 1205 const ColumnRef column{table, &info, index};
459 1205 CloneRowsParams params{};
460
1/1
✓ Branch 1 taken 1205 times.
 1205 params.column.address = rows.Pages(index);
461
1/1
✓ Branch 1 taken 1205 times.
 1205 params.sources.address = sourceRows->Address();
462 1205 params.first = first;
463 1205 params.count = count;
464 1205 params.copies = copies;
465
1/1
✓ Branch 1 taken 1205 times.
 1205 params.words = rows.CellBytes(index) / sizeof(uint32_t);
466
1/1
✓ Branch 1 taken 1205 times.
 1205 Dispatch("CloneRows", params, uint64_t(copies) * count * params.words);
467
1/1
✓ Branch 1 taken 1205 times.
 1205 Touch(column);
468
1/1
✓ Branch 1 taken 1205 times.
 1205 StampRows(column, first, count * copies);
469
2/4
✓ Branch 0 taken 1205 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1205 times.
✗ Branch 3 not taken.
 1205 for (uint32_t source = 0; source < count && !m_overrides.empty(); ++source) {
470 ✗ const auto found = m_overrides.find(ArrayKey(column, sources[source]));
471
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (found == m_overrides.end()) {
472 ✗ continue;
473 }
474
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ for (uint32_t copy = 0; copy < copies; ++copy) {
475
0/3
✗ Branch 1 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 ✗ StageLive(column, first + copy * count + source, found->second.authored);
476 }
477 }
478 }
479 1014 }
480
481 273526 uint64_t ColumnStore::ArrayKey(ColumnRef column, uint32_t row) {
482 273526 return (uint64_t(column.schema) << 48) | (uint64_t(column.index) << 32) | row;
483 }
484
485 11458 ArrayRef ColumnStore::Array(ColumnRef column, uint32_t row) const {
486 11458 const auto found = m_arrays.find(ArrayKey(column, row));
487
2/2
✓ Branch 0 taken 5553 times.
✓ Branch 1 taken 5905 times.
 17363 return found == m_arrays.end() ? ArrayRef{} : found->second;
488 }
489
490 122 bool ColumnStore::Exclusive(ColumnRef column, uint32_t row) const {
491 122 const ArrayRef array = Array(column, row);
492
1/2
✓ Branch 0 taken 122 times.
✗ Branch 1 not taken.
 122 if (array.address == 0) {
493 return false;
494 }
495 122 const uint32_t rows[] = {row};
496
3/4
✓ Branch 1 taken 121 times.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
✓ Branch 4 taken 121 times.
 243 return Attached(column, rows) ||
497
2/2
✓ Branch 1 taken 121 times.
✓ Branch 4 taken 121 times.
 121 const_cast<ColumnStore&>(*this).Arrays(column.column->type).Borrowed(array);
498 }
499
500 2578 void ColumnStore::SetArray(ColumnRef column, uint32_t row, const ArrayRef& array) {
501
2/2
✓ Branch 0 taken 2166 times.
✓ Branch 1 taken 412 times.
 2578 CheckStoreLock(m_lock, "Writing a scene's columns");
502 2578 const uint64_t key = ArrayKey(column, row);
503
2/2
✓ Branch 1 taken 396 times.
✓ Branch 2 taken 2182 times.
 2578 if (const auto found = m_arrays.find(key); found != m_arrays.end()) {
504 396 const ArrayRef old = found->second;
505 396 m_arrays.erase(found);
506
2/2
✓ Branch 0 taken 298 times.
✓ Branch 1 taken 98 times.
 396 if (old.address != array.address) {
507
2/2
✓ Branch 1 taken 298 times.
✓ Branch 4 taken 298 times.
 298 Arrays(column.column->type).Free(old);
508 }
509 }
510
2/2
✓ Branch 0 taken 2482 times.
✓ Branch 1 taken 96 times.
 2578 if (array.address != 0) {
511 2482 m_arrays[key] = array;
512 }
513 2578 Stage(column, row, std::as_bytes(std::span(&array, 1)));
514 2578 m_arrayWrites[ArrayKey(column, 0)].push_back({m_stamp, row});
515 2578 }
516
517 61321 std::vector<uint32_t> ColumnStore::ArrayRows(ColumnRef column, Ordinal since, Ordinal until) const {
518 61321 std::vector<uint32_t> rows;
519 61321 const auto found = m_arrayWrites.find(ArrayKey(column, 0));
520
2/2
✓ Branch 0 taken 5721 times.
✓ Branch 1 taken 55600 times.
 61321 if (found == m_arrayWrites.end()) {
521 return rows;
522 }
523 5721 const std::vector<ArrayWrite>& writes = found->second;
524 11276 for (auto at = std::ranges::upper_bound(writes, since, {}, &ArrayWrite::stamp);
525
3/4
✓ Branch 0 taken 5555 times.
✓ Branch 1 taken 5721 times.
✓ Branch 2 taken 5555 times.
✗ Branch 3 not taken.
 11276 at != writes.end() && at->stamp <= until; ++at) {
526
1/1
✓ Branch 1 taken 5555 times.
 5555 rows.push_back(at->row);
527 }
528 5721 std::ranges::sort(rows);
529 5721 rows.erase(std::ranges::unique(rows).begin(), rows.end());
530 5721 return rows;
531 ✗ }
532
533 23 void ColumnStore::LogArrayRows(ColumnRef column, std::span<const uint32_t> rows) {
534 23 std::vector<ArrayWrite>& writes = m_arrayWrites[ArrayKey(column, 0)];
535
2/2
✓ Branch 0 taken 47 times.
✓ Branch 1 taken 23 times.
 70 for (const uint32_t row : rows) {
536
1/2
✓ Branch 0 taken 47 times.
✗ Branch 1 not taken.
 47 if (row != kNoRow) {
537 47 writes.push_back({m_stamp, row});
538 }
539 }
540 23 }
541
542 1166 void ColumnStore::TrimArrayWrites(Ordinal through,
543 std::span<const std::pair<ColumnRef, Ordinal>> held) {
544
2/2
✓ Branch 0 taken 3629 times.
✓ Branch 1 taken 1166 times.
 4795 for (auto& [key, writes] : m_arrayWrites) {
545 3629 Ordinal kept = through;
546
4/4
✓ Branch 0 taken 3628 times.
✓ Branch 1 taken 191906 times.
✓ Branch 2 taken 195534 times.
✓ Branch 3 taken 3629 times.
 199163 for (const auto& [column, seen] : held) {
547
2/2
✓ Branch 0 taken 3628 times.
✓ Branch 1 taken 191906 times.
 195534 if (ArrayKey(column, 0) == key) {
548
2/2
✓ Branch 0 taken 1464 times.
✓ Branch 1 taken 2164 times.
 5092 kept = std::min(kept, seen);
549 }
550 }
551 3629 writes.erase(writes.begin(),
552 3629 std::ranges::upper_bound(writes, kept, {}, &ArrayWrite::stamp));
553 }
554 1166 }
555
556 2720 void ColumnStore::SetCounts(uint32_t prims, uint32_t worlds) {
557 2720 m_primCount = prims;
558 2720 m_worldCount = worlds;
559
2/2
✓ Branch 0 taken 138 times.
✓ Branch 1 taken 2582 times.
 2720 if (worlds + 1 > m_worlds.Rows()) {
560 138 m_worlds.Allocate(worlds + 1 - m_worlds.Rows());
561 }
562 2720 m_worldStamps.resize(worlds + 1, 0);
563 2720 }
564
565 820325 void ColumnStore::Place(std::byte* data, uint32_t offset, uint64_t address) const {
566 820325 std::memcpy(data + offset, &address, sizeof(address));
567 820325 }
568
569 134833 void ColumnStore::Flush() {
570
2/2
✓ Branch 0 taken 133543 times.
✓ Branch 1 taken 1290 times.
 134833 CheckStoreLock(m_lock, "Writing a scene's columns");
571
4/5
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 134827 times.
✓ Branch 3 taken 6 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 6 times.
 134833 CHRONO_BATCH_RENDER_ZONE("ColumnStore/Flush");
572
1/1
✓ Branch 1 taken 134833 times.
 134833 LandAuthored(false);
573
574 134833 struct Scattered {
575 ColumnTable* table;
576 uint32_t column;
577 uint32_t count;
578 uint32_t words;
579 uint64_t rows;
580 uint64_t cells;
581 };
582
583 134833 std::vector<Scattered> scattered;
584 134833 std::vector<std::byte>& payload = m_payload;
585 134833 payload.clear();
586 15414965 const auto flush = [&](ColumnTable& table) {
587
1/1
✓ Branch 1 taken 15280132 times.
 15280132 table.Flush(
588 15280132 [&](uint32_t column, std::span<const uint32_t> rows, std::span<const std::byte> cells) {
589 3 const uint64_t at = payload.size();
590 3 const std::span<const std::byte> rowBytes = std::as_bytes(rows);
591 3 payload.insert(payload.end(), rowBytes.begin(), rowBytes.end());
592 3 payload.insert(payload.end(), cells.begin(), cells.end());
593 3 scattered.push_back({&table, column, uint32_t(rows.size()),
594 3 uint32_t(cells.size() / rows.size() / sizeof(uint32_t)), at,
595 3 at + rowBytes.size()});
596 3 });
597 15280132 };
598
2/2
✓ Branch 0 taken 14875633 times.
✓ Branch 1 taken 134833 times.
 15010466 for (Entry& entry : m_entries) {
599
1/1
✓ Branch 1 taken 14875633 times.
 14875633 flush(*entry.table);
600 }
601
1/1
✓ Branch 1 taken 134833 times.
 134833 flush(m_prims);
602
1/1
✓ Branch 1 taken 134833 times.
 134833 flush(m_slots);
603
1/1
✓ Branch 1 taken 134833 times.
 134833 flush(m_worlds);
604 134833 std::vector<CopyRun> runs;
605 134833 uint32_t total = 0;
606 134833 device::Queue& queue = m_device.GetQueue();
607
4/4
✓ Branch 0 taken 143865 times.
✓ Branch 1 taken 376 times.
✓ Branch 2 taken 144241 times.
✓ Branch 3 taken 134833 times.
 279074 for (const auto& [type, heap] : m_arrayHeaps) {
608
2/2
✓ Branch 0 taken 143865 times.
✓ Branch 1 taken 376 times.
 144241 const std::span<const std::byte> staged = heap->StagedBytes();
609
2/2
✓ Branch 0 taken 143865 times.
✓ Branch 1 taken 376 times.
 144241 if (staged.empty()) {
610 143865 continue;
611 }
612
1/1
✓ Branch 1 taken 376 times.
 376 const uint64_t base = queue.Upload(staged.data(), staged.size()).Address();
613
3/3
✓ Branch 1 taken 376 times.
✓ Branch 3 taken 685 times.
✓ Branch 4 taken 376 times.
 1061 for (const ArrayPool::StagedCopy& copy : heap->TakeStaged(base)) {
614
1/1
✓ Branch 1 taken 685 times.
 685 runs.push_back({copy.dst, copy.source, total, 0});
615 685 total += copy.words;
616 376 }
617 }
618
2/2
✓ Branch 0 taken 372 times.
✓ Branch 1 taken 134461 times.
 134833 if (!runs.empty()) {
619
1/1
✓ Branch 1 taken 372 times.
 372 const std::span<const std::byte> table = std::as_bytes(std::span(runs));
620
1/1
✓ Branch 1 taken 372 times.
 372 Dispatch("CopyWords",
621
2/2
✓ Branch 1 taken 372 times.
✓ Branch 4 taken 372 times.
 372 CopyWordsParams{.runs = {queue.Upload(table.data(), table.size()).Address()},
622
1/1
✓ Branch 1 taken 372 times.
 372 .runCount = uint32_t(runs.size()),
623 .total = total},
624 total);
625 }
626
2/2
✓ Branch 0 taken 134832 times.
✓ Branch 1 taken 1 time.
 134833 if (scattered.empty()) {
627 134832 return;
628 }
629 1 const Ref<device::Buffer> uploaded =
630
2/2
✓ Branch 1 taken 1 time.
✓ Branch 4 taken 1 time.
 1 m_device.CreateBuffer({.label = "scattered cells"}, std::span<const std::byte>(payload));
631
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 1 time.
 4 for (const Scattered& scatter : scattered) {
632 3 CellsParams params{};
633
1/1
✓ Branch 1 taken 3 times.
 3 params.column.address = scatter.table->Pages(scatter.column);
634
1/1
✓ Branch 1 taken 3 times.
 3 params.rows.address = uploaded->Address() + scatter.rows;
635 3 params.cells.address = uploaded->Address() + scatter.cells;
636 3 params.count = scatter.count;
637 3 params.words = scatter.words;
638
1/1
✓ Branch 1 taken 3 times.
 3 Dispatch("ScatterCells", params, uint64_t(scatter.count) * scatter.words);
639 }
640 134835 }
641
642 1727 AuthoredData ColumnStore::Data() const {
643 1727 AuthoredData data{};
644 1727 data.prims.schema.pages.address = m_prims.Pages(kPrimSchema);
645 1727 data.prims.parent.pages.address = m_prims.Pages(kPrimParent);
646 1727 data.prims.world.pages.address = m_prims.Pages(kPrimWorld);
647 1727 data.prims.flags.pages.address = m_prims.Pages(kPrimFlags);
648 1727 data.prims.slots.pages.address = m_prims.Pages(kPrimSlots);
649 1727 data.prims.apis.pages.address = m_prims.Pages(kPrimApis);
650 1727 data.rows.pages.address = m_slots.Pages(0);
651 1727 data.worldStamps.pages.address = m_worlds.Pages(0);
652
653 1727 std::byte* schemas = reinterpret_cast<std::byte*>(&data.schemas);
654 1727 uint32_t base = 0;
655
2/2
✓ Branch 0 taken 189970 times.
✓ Branch 1 taken 1727 times.
 191697 for (uint32_t id = 0; id < uint32_t(SchemaId::Count); ++id) {
656 189970 const SchemaId schema = SchemaId(id);
657 189970 const SchemaInfo& info = Info(schema);
658 189970 const Entry& entry = m_entries[id];
659 189970 const Span<const ColumnInfo> columns = Columns(schema);
660
2/2
✓ Branch 0 taken 630355 times.
✓ Branch 1 taken 189970 times.
 820325 for (uint32_t column = 0; column < Info(schema).columnCount; ++column) {
661 630355 const ColumnInfo& at = columns[column];
662 630355 Place(schemas + base, at.offset, entry.table->Pages(column));
663 }
664 189970 Place(schemas + base, info.dataBytes - sizeof(uint64_t),
665 189970 entry.table->Pages(Info(schema).columnCount));
666 189970 base += info.dataBytes;
667 }
668 1727 data.primCount = m_primCount;
669 1727 data.worldCount = m_worldCount;
670 1727 return data;
671 }
672
673 80969 uint64_t ColumnStore::DeviceData() {
674
4/5
✓ Branch 0 taken 5 times.
✓ Branch 1 taken 80964 times.
✓ Branch 3 taken 5 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 5 times.
 80969 CHRONO_BATCH_RENDER_ZONE("ColumnStore/DeviceData");
675
1/1
✓ Branch 1 taken 80969 times.
 80969 Flush();
676
2/2
✓ Branch 0 taken 402 times.
✓ Branch 1 taken 80567 times.
 80969 if (!m_deviceData) {
677 402 m_deviceData =
678
2/3
✓ Branch 1 taken 402 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 402 times.
 402 m_device.CreateBuffer({.bytes = sizeof(AuthoredData), .label = "scene data"});
679 }
680 80969 std::vector<uint64_t> layout = {m_primCount, m_worldCount, m_prims.Layout(), m_slots.Layout(),
681
1/1
✓ Branch 1 taken 80969 times.
 80969 m_worlds.Layout()};
682
2/2
✓ Branch 0 taken 8931636 times.
✓ Branch 1 taken 80969 times.
 9012605 for (const Entry& entry : m_entries) {
683
1/1
✓ Branch 1 taken 8931636 times.
 8931636 layout.push_back(uint64_t(reinterpret_cast<uintptr_t>(entry.table.get())));
684
2/3
✓ Branch 0 taken 8931636 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 8931636 times.
 8931636 layout.push_back(entry.table ? entry.table->Layout() : 0);
685 }
686
2/2
✓ Branch 0 taken 80412 times.
✓ Branch 1 taken 557 times.
 80969 if (layout == m_uploadedLayout) {
687 80412 return m_deviceData->Address();
688 }
689
1/1
✓ Branch 1 taken 557 times.
 557 const AuthoredData data = Data();
690
2/2
✓ Branch 0 taken 155 times.
✓ Branch 1 taken 402 times.
 557 const std::span<const std::byte> bytes = std::as_bytes(std::span(&data, 1));
691
2/2
✓ Branch 0 taken 155 times.
✓ Branch 1 taken 402 times.
 557 if (m_uploaded.size() != bytes.size() ||
692
2/2
✓ Branch 0 taken 104 times.
✓ Branch 1 taken 51 times.
 155 std::memcmp(m_uploaded.data(), bytes.data(), bytes.size()) != 0) {
693
1/1
✓ Branch 1 taken 506 times.
 506 m_device.GetQueue().Write(*m_deviceData, &data);
694
1/1
✓ Branch 1 taken 506 times.
 506 m_uploaded.assign(bytes.begin(), bytes.end());
695 }
696 557 m_uploadedLayout = std::move(layout);
697 557 return m_deviceData->Address();
698 81773 }
699
700 }
701