GCC Code Coverage Report

chrono_batch_render/
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
269 / 288
93.4%
Branches:
212 / 264
80.3%
Functions:
23 / 23
100.0%
stage/columns/column_table.cpp
Line Branch Exec Source
1 // SPDX-License-Identifier: BSD-3-Clause
2 #include "chrono_batch_render/stage/columns/column_table.h"
3
4 #include <algorithm>
5 #include <utility>
6
7 #include <fmt/format.h>
8
9 #include "chrono_batch_render/device/commands.h"
10 #include "chrono_batch_render/device/kernels/shared.h"
11 #include "chrono_batch_render/device/queue.h"
12 #include "chrono_batch_render/stage/columns/kernels/paged.h"
13 #include "chrono_batch_render/util/error.h"
14 #include "chrono_batch_render/util/profiler.h"
15 #include "chrono_batch_render/util/span.h"
16
17 namespace chrono::batch_render {
18
19 namespace {
20
21 constexpr uint32_t kLargestSpan = 64 * 1024;
22 constexpr uint64_t kSpanAlignment = 256;
23
24 }
25
26 57870 ColumnTable::ColumnTable(DeviceHeap& heap, std::string label, std::vector<TableColumn> columns)
27
1/1
✓ Branch 2 taken 57870 times.
 57870 : m_heap(heap), m_label(std::move(label)), m_columns(std::move(columns)),
28
3/3
✓ Branch 1 taken 57870 times.
✓ Branch 4 taken 57870 times.
✓ Branch 7 taken 57870 times.
 57870 m_pages(m_columns.size()), m_staged(m_columns.size()), m_redirections(m_columns.size()) {
29
2/2
✓ Branch 0 taken 246417 times.
✓ Branch 1 taken 57870 times.
 304287 for (const TableColumn& column : m_columns) {
30
2/4
✓ Branch 0 taken 246417 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 246417 times.
 246417 if (column.cellBytes == 0 || column.cellBytes % 4 != 0) {
31 ✗ throw Error("{} has a column of {} bytes; cells are whole words", m_label,
32
0/1
✗ Branch 2 not taken.
 ✗ column.cellBytes);
33 }
34 }
35 57870 }
36
37 1699683 uint32_t ColumnTable::CellBytes(uint32_t column) const {
38
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1699683 times.
 1699683 if (column >= m_columns.size()) {
39
0/1
✗ Branch 2 not taken.
 ✗ throw Error("{} has no column {}", m_label, column);
40 }
41 1699683 return m_columns[column].cellBytes;
42 }
43
44 5688 uint32_t ColumnTable::AddColumn(TableColumn column) {
45
2/4
✓ Branch 0 taken 5688 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 5688 times.
 5688 if (column.cellBytes == 0 || column.cellBytes % 4 != 0) {
46 ✗ throw Error("{} has a column of {} bytes; cells are whole words", m_label,
47
0/1
✗ Branch 2 not taken.
 ✗ column.cellBytes);
48 }
49 5688 m_columns.push_back(column);
50 5688 m_pages.emplace_back();
51 5688 m_staged.emplace_back();
52 5688 m_redirections.emplace_back();
53
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 5686 times.
 5688 if (!m_spans.empty()) {
54 2 device::Queue& queue = m_heap.GetDevice().GetQueue();
55 2 device::CommandEncoder encoder = queue.CreateEncoder();
56
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 2 times.
 4 for (Span& span : m_spans) {
57 2 const HeapBlock block =
58
1/1
✓ Branch 1 taken 2 times.
 2 m_heap.Allocate(uint64_t(span.rows) * column.cellBytes, kSpanAlignment);
59
2/2
✓ Branch 1 taken 2 times.
✓ Branch 4 taken 2 times.
 2 encoder.Fill(block.Range(), column.fill);
60
2/2
✓ Branch 0 taken 3 times.
✓ Branch 1 taken 2 times.
 5 for (uint32_t page = 0; page < span.rows / kPageRows; ++page) {
61
1/1
✓ Branch 1 taken 3 times.
 3 m_pages.back().push_back(block.Address() +
62
1/1
✓ Branch 1 taken 3 times.
 3 uint64_t(page) * kPageRows * column.cellBytes);
63 }
64
1/1
✓ Branch 1 taken 2 times.
 2 span.blocks.push_back(block);
65 }
66
2/2
✓ Branch 1 taken 2 times.
✓ Branch 4 taken 2 times.
 2 queue.Submit(encoder.Finish());
67 2 m_pageCapacity = 0;
68
1/1
✓ Branch 1 taken 2 times.
 2 WritePages(0, m_capacity / kPageRows);
69 2 }
70 5688 return uint32_t(m_columns.size() - 1);
71 }
72
73 16737 uint32_t ColumnTable::Allocate(uint32_t count, bool pageAligned) {
74
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 16737 times.
 16737 if (count == 0) {
75 ✗ return m_rows;
76 }
77
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 16730 times.
 16737 if (pageAligned) {
78
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 const uint32_t aligned = std::min(uint32_t(RoundUp(m_rows, kPageRows)), m_capacity);
79
2/2
✓ Branch 0 taken 497 times.
✓ Branch 1 taken 7 times.
 504 for (uint32_t row = m_rows; row < aligned; ++row) {
80 497 m_free.push_back(row);
81 }
82 7 m_rows = aligned;
83 }
84
2/2
✓ Branch 0 taken 6340 times.
✓ Branch 1 taken 10397 times.
 16737 if (uint64_t(m_rows) + count > m_capacity) {
85
2/2
✓ Branch 0 taken 6599 times.
✓ Branch 1 taken 6340 times.
 12939 for (uint32_t row = m_rows; row < m_capacity; ++row) {
86 6599 m_free.push_back(row);
87 }
88 6340 m_rows = m_capacity;
89 6340 Grow(count);
90 }
91 16737 const uint32_t first = m_rows;
92 16737 m_rows += count;
93
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 16730 times.
 16737 if (pageAligned) {
94
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 7 times.
 7 m_rows = std::min(uint32_t(RoundUp(m_rows, kPageRows)), m_capacity);
95 7 m_aligned[first] = count;
96 }
97 16737 return first;
98 }
99
100 10647 uint32_t ColumnTable::Acquire() {
101
2/2
✓ Branch 0 taken 9840 times.
✓ Branch 1 taken 807 times.
 10647 if (m_free.empty()) {
102 9840 return Allocate(1);
103 }
104 807 const uint32_t row = m_free.back();
105 807 m_free.pop_back();
106 807 return row;
107 }
108
109 1217 void ColumnTable::Release(uint32_t row) {
110 1217 CheckRows(row, 1);
111 1216 const auto aligned = m_aligned.upper_bound(row);
112
3/4
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 1215 times.
✓ Branch 2 taken 1 time.
✗ Branch 3 not taken.
 1217 if (aligned != m_aligned.begin() &&
113
1/2
✓ Branch 1 taken 1 time.
✗ Branch 2 not taken.
 1 row < std::prev(aligned)->first + std::prev(aligned)->second) {
114 1 return;
115 }
116 1215 m_free.push_back(row);
117 }
118
119 6340 void ColumnTable::Grow(uint32_t count) {
120
4/5
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 6334 times.
✓ Branch 3 taken 6 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 6 times.
 6340 CHRONO_BATCH_RENDER_ZONE("ColumnTable/Grow");
121 12680 const uint32_t rows = uint32_t(RoundUp(
122
6/7
✓ Branch 0 taken 6294 times.
✓ Branch 1 taken 46 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 6340 times.
✓ Branch 4 taken 6280 times.
✓ Branch 5 taken 60 times.
✓ Branch 7 taken 6340 times.
 18914 std::max(count, std::min(std::max(m_capacity, kPageRows), kLargestSpan)), kPageRows));
123 6340 Span span{m_capacity, rows, {}};
124
1/1
✓ Branch 1 taken 6340 times.
 6340 device::Queue& queue = m_heap.GetDevice().GetQueue();
125
1/1
✓ Branch 1 taken 6340 times.
 6340 device::CommandEncoder encoder = queue.CreateEncoder();
126
2/2
✓ Branch 0 taken 34970 times.
✓ Branch 1 taken 6340 times.
 41310 for (uint32_t column = 0; column < m_columns.size(); ++column) {
127
1/1
✓ Branch 1 taken 34970 times.
 34970 const TableColumn& info = m_columns[column];
128
1/1
✓ Branch 1 taken 34970 times.
 34970 const HeapBlock block = m_heap.Allocate(uint64_t(rows) * info.cellBytes, kSpanAlignment);
129
2/2
✓ Branch 1 taken 34970 times.
✓ Branch 4 taken 34970 times.
 34970 encoder.Fill(block.Range(), info.fill);
130
2/2
✓ Branch 0 taken 37860 times.
✓ Branch 1 taken 34970 times.
 72830 for (uint32_t page = 0; page < rows / kPageRows; ++page) {
131
1/1
✓ Branch 1 taken 37860 times.
 37860 m_pages[column].push_back(block.Address() +
132
1/1
✓ Branch 1 taken 37860 times.
 37860 uint64_t(page) * kPageRows * info.cellBytes);
133 }
134
1/1
✓ Branch 1 taken 34970 times.
 34970 span.blocks.push_back(block);
135 }
136
2/2
✓ Branch 1 taken 6340 times.
✓ Branch 4 taken 6340 times.
 6340 queue.Submit(encoder.Finish());
137
138 6340 const uint32_t firstPage = m_capacity / kPageRows;
139
1/1
✓ Branch 1 taken 6340 times.
 6340 m_spans.push_back(std::move(span));
140 6340 m_capacity += rows;
141
1/1
✓ Branch 1 taken 6340 times.
 6340 WritePages(firstPage, rows / kPageRows);
142 6340 }
143
144 6344 void ColumnTable::WritePages(uint32_t firstPage, uint32_t pageCount) {
145 6344 const uint32_t pages = m_capacity / kPageRows;
146
2/2
✓ Branch 0 taken 6342 times.
✓ Branch 1 taken 2 times.
 6344 device::Queue& queue = m_heap.GetDevice().GetQueue();
147
2/2
✓ Branch 0 taken 6342 times.
✓ Branch 1 taken 2 times.
 6344 if (pages > m_pageCapacity) {
148
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 6342 times.
 6342 m_pageCapacity = std::max(pages, m_pageCapacity * 2);
149
1/1
✓ Branch 2 taken 6342 times.
 12684 m_pageTable = m_heap.GetDevice().CreateBuffer(
150 6342 {.bytes = uint64_t(m_pageCapacity) * m_columns.size() * sizeof(uint64_t),
151
2/2
✓ Branch 1 taken 48 times.
✓ Branch 2 taken 6294 times.
 12684 .label = fmt::format("{} pages", m_label)});
152 6342 ++m_layout;
153 6342 firstPage = 0;
154 6342 pageCount = pages;
155 }
156 6344 const uint64_t stride = uint64_t(m_pageCapacity) * sizeof(uint64_t);
157
2/2
✓ Branch 0 taken 34978 times.
✓ Branch 1 taken 6344 times.
 41322 for (uint32_t column = 0; column < m_columns.size(); ++column) {
158 34978 queue.Write(m_pageTable->Slice(column * stride + firstPage * sizeof(uint64_t),
159 34978 uint64_t(pageCount) * sizeof(uint64_t)),
160 34978 m_pages[column].data() + firstPage);
161 }
162 12686 }
163
164 1062152 uint64_t ColumnTable::Pages(uint32_t column) const {
165 1062152 CellBytes(column);
166
2/2
✓ Branch 0 taken 363596 times.
✓ Branch 1 taken 698556 times.
 1062152 return m_pageTable
167
2/2
✓ Branch 0 taken 363596 times.
✓ Branch 1 taken 698556 times.
 1062152 ? m_pageTable->Address() + uint64_t(column) * m_pageCapacity * sizeof(uint64_t)
168 1062152 : 0;
169 }
170
171 92600 const ColumnTable::Redirection* ColumnTable::FindRedirection(uint32_t column, uint32_t row) const {
172
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 92598 times.
 92600 for (const Redirection& redirection : m_redirections[column]) {
173
2/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✗ Branch 3 not taken.
 2 if (row >= redirection.first && row < redirection.first + redirection.rows) {
174 2 return &redirection;
175 }
176 }
177 92598 return nullptr;
178 }
179
180 2 void ColumnTable::PointPages(uint32_t column, uint32_t first, uint32_t count, uint64_t address) {
181 2 const uint32_t firstPage = first / kPageRows;
182 2 const uint32_t pageCount = (count + kPageRows - 1) / kPageRows;
183 2 const uint64_t pageBytes = uint64_t(kPageRows) * m_columns[column].cellBytes;
184
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 2 times.
 6 for (uint32_t page = 0; page < pageCount; ++page) {
185 4 m_pages[column][firstPage + page] = address + page * pageBytes;
186 }
187 2 WritePages(firstPage, pageCount);
188 2 }
189
190 3 void ColumnTable::Redirect(uint32_t column, uint32_t first, uint32_t count,
191 Ref<device::Buffer> buffer) {
192 3 const uint32_t cell = CellBytes(column);
193 3 CheckRows(first, count);
194
2/4
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 3 times.
 3 if (count == 0 || first % kPageRows != 0) {
195 ✗ throw Error("Rows {} to {} of {} don't start a page, so they can't be redirected", first,
196
0/1
✗ Branch 2 not taken.
 ✗ first + count, m_label);
197 }
198 3 const auto aligned = m_aligned.upper_bound(first);
199
3/4
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 1 time.
✗ Branch 2 not taken.
✓ Branch 3 taken 2 times.
 5 const bool inside = aligned != m_aligned.begin() &&
200
1/2
✗ Branch 2 not taken.
✓ Branch 3 taken 2 times.
 2 first + count <= std::prev(aligned)->first + std::prev(aligned)->second;
201 3 const bool wholePages =
202
3/4
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
✓ Branch 3 taken 1 time.
 3 (first + count) % kPageRows == 0 ||
203
2/2
✓ Branch 1 taken 1 time.
✓ Branch 2 taken 1 time.
 2 (inside && first + count == std::prev(aligned)->first + std::prev(aligned)->second);
204
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 1 time.
 3 if (!inside || !wholePages) {
205 2 throw Error("Rows {} to {} of {} weren't allocated on whole pages of their own", first,
206
1/1
✓ Branch 2 taken 2 times.
 2 first + count, m_label);
207 }
208
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
 1 if (Find(first).first + Find(first).rows < first + count) {
209
0/1
✗ Branch 2 not taken.
 ✗ throw Error("Rows {} to {} of {} cross a span", first, first + count, m_label);
210 }
211
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1 time.
 1 if (buffer->Bytes() < uint64_t(count) * cell) {
212
0/1
✗ Branch 2 not taken.
 ✗ throw Error("{} bytes for {} rows of {}-byte cells", buffer->Bytes(), count, cell);
213 }
214
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 1 time.
 3 for (uint32_t row = first; row < first + count; row += kPageRows) {
215
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 2 times.
 2 if (FindRedirection(column, row)) {
216 ✗ throw Error("Rows {} to {} of {} are already redirected", first, first + count,
217
0/1
✗ Branch 2 not taken.
 ✗ m_label);
218 }
219 }
220
1/1
✓ Branch 1 taken 1 time.
 1 Flush();
221 1 PointPages(column, first, count, buffer->Address());
222
2/3
✓ Branch 1 taken 1 time.
✗ Branch 3 not taken.
✓ Branch 4 taken 1 time.
 1 m_redirections[column].push_back({first, count, std::move(buffer)});
223 1 ++m_layout;
224 2 }
225
226 2 Ref<device::Buffer> ColumnTable::Restore(uint32_t column, uint32_t first) {
227 2 CellBytes(column);
228 2 std::vector<Redirection>& redirections = m_redirections[column];
229 2 const auto found = std::ranges::find(redirections, first, &Redirection::first);
230
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 1 time.
 2 if (found == redirections.end()) {
231
1/1
✓ Branch 2 taken 1 time.
 1 throw Error("Rows from {} of {} aren't redirected", first, m_label);
232 }
233
1/1
✓ Branch 1 taken 1 time.
 1 Flush();
234 1 const Redirection redirection = *found;
235 1 redirections.erase(found);
236 1 const Span& span = Find(first);
237 1 PointPages(column, first, redirection.rows,
238
1/1
✓ Branch 1 taken 1 time.
 1 span.blocks[column].Address() +
239
1/1
✓ Branch 1 taken 1 time.
 1 uint64_t(first - span.first) * m_columns[column].cellBytes);
240 1 ++m_layout;
241
1/2
✓ Branch 1 taken 1 time.
✗ Branch 2 not taken.
 1 return redirection.buffer;
242 1 }
243
244 2 bool ColumnTable::Redirected(uint32_t column, uint32_t first, uint32_t count) const {
245 2 CellBytes(column);
246 2 return std::ranges::any_of(m_redirections[column], [&](const Redirection& r) {
247
3/4
✓ Branch 0 taken 2 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 1 time.
✓ Branch 3 taken 1 time.
 2 return first < r.first + r.rows && r.first < first + count;
248 2 });
249 }
250
251 92598 const ColumnTable::Span& ColumnTable::Find(uint32_t row) const {
252 92598 const auto after = std::ranges::upper_bound(m_spans, row, {}, &Span::first);
253 92598 return *(after - 1);
254 }
255
256 535289 void ColumnTable::CheckRows(uint32_t first, uint32_t count) const {
257
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 535288 times.
 535289 if (uint64_t(first) + count > m_rows) {
258 3 throw Error("Rows {} to {} of {}, which has {}", first, uint64_t(first) + count, m_label,
259
1/1
✓ Branch 2 taken 1 time.
 1 m_rows);
260 }
261 535288 }
262
263 93701 std::vector<device::BufferRange> ColumnTable::Ranges(uint32_t column, uint32_t first,
264 uint32_t count) const {
265 93701 const uint64_t cell = CellBytes(column);
266 93701 CheckRows(first, count);
267 93701 std::vector<device::BufferRange> ranges;
268
2/2
✓ Branch 0 taken 92598 times.
✓ Branch 1 taken 93701 times.
 186299 while (count > 0) {
269
2/2
✓ Branch 1 taken 2 times.
✓ Branch 2 taken 92596 times.
 92598 if (const Redirection* redirection = FindRedirection(column, first)) {
270
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 const uint32_t taken = std::min(count, redirection->first + redirection->rows - first);
271
1/1
✓ Branch 1 taken 2 times.
 2 ranges.push_back(redirection->buffer->Slice(uint64_t(first - redirection->first) * cell,
272
1/1
✓ Branch 1 taken 2 times.
 2 uint64_t(taken) * cell));
273 2 first += taken;
274 2 count -= taken;
275 2 continue;
276 2 }
277 92596 const Span& span = Find(first);
278
2/2
✓ Branch 0 taken 9 times.
✓ Branch 1 taken 92587 times.
 92596 uint32_t taken = std::min(count, span.first + span.rows - first);
279
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 92596 times.
 92596 for (const Redirection& redirection : m_redirections[column]) {
280
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (redirection.first > first) {
281
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ taken = std::min(taken, redirection.first - first);
282 }
283 }
284 185192 ranges.push_back(
285
2/2
✓ Branch 1 taken 92596 times.
✓ Branch 4 taken 92596 times.
 92596 span.blocks[column].Range().Slice((first - span.first) * cell, taken * cell));
286 92596 first += taken;
287 92596 count -= taken;
288 }
289 93701 return ranges;
290 ✗ }
291
292 12719 std::optional<device::BufferRange> ColumnTable::Contiguous(uint32_t column, uint32_t first,
293 uint32_t count) const {
294
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 12719 times.
 12719 if (count == 0) {
295 ✗ return std::nullopt;
296 }
297 12719 const std::vector<device::BufferRange> ranges = Ranges(column, first, count);
298
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 12718 times.
 12719 if (ranges.size() != 1) {
299 1 return std::nullopt;
300 }
301 12718 return ranges.front();
302 12719 }
303
304 440368 void ColumnTable::Stage(uint32_t column, uint32_t first, std::span<const std::byte> bytes) {
305 440368 const uint32_t cell = CellBytes(column);
306
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 440368 times.
 440368 if (bytes.size() % cell != 0) {
307
0/1
✗ Branch 1 not taken.
 ✗ throw Error("{} bytes for column {} of {}, whose cells are {} bytes", bytes.size(), column,
308
0/1
✗ Branch 2 not taken.
 ✗ m_label, cell);
309 }
310 440368 const uint32_t count = uint32_t(bytes.size() / cell);
311 440368 CheckRows(first, count);
312 440368 Staging& staged = m_staged[column];
313
2/2
✓ Branch 0 taken 781988 times.
✓ Branch 1 taken 440368 times.
 1222356 for (uint32_t i = 0; i < count; ++i) {
314 781988 staged.rows.push_back(first + i);
315 }
316 440368 staged.bytes.insert(staged.bytes.end(), bytes.begin(), bytes.end());
317
2/2
✓ Branch 0 taken 59 times.
✓ Branch 1 taken 440309 times.
 440368 if (staged.rows.size() >= staged.coalesceAt) {
318 59 Coalesce(staged, cell);
319
1/2
✓ Branch 0 taken 59 times.
✗ Branch 1 not taken.
 118 staged.coalesceAt = std::max<size_t>(4096, 2 * staged.rows.size());
320 }
321 440368 }
322
323 78131 void ColumnTable::Coalesce(Staging& staged, uint32_t cell) {
324 78131 std::vector<uint32_t> order(staged.rows.size());
325
2/2
✓ Branch 0 taken 987747 times.
✓ Branch 1 taken 78131 times.
 1065878 for (uint32_t i = 0; i < order.size(); ++i) {
326 987747 order[i] = i;
327 }
328
8/12
✓ Branch 0 taken 44757 times.
✓ Branch 1 taken 1839257 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✓ Branch 6 taken 26040 times.
✓ Branch 7 taken 764320 times.
✓ Branch 8 taken 10804 times.
✓ Branch 9 taken 31937 times.
✓ Branch 10 taken 9552 times.
✓ Branch 11 taken 428613 times.
 3233411 std::ranges::stable_sort(order, {}, [&](uint32_t i) { return staged.rows[i]; });
329 78131 std::vector<uint32_t> rows;
330 78131 std::vector<std::byte> bytes;
331
2/2
✓ Branch 0 taken 987747 times.
✓ Branch 1 taken 78131 times.
 1065878 for (uint32_t at = 0; at < order.size(); ++at) {
332
2/2
✓ Branch 0 taken 909616 times.
✓ Branch 1 taken 78131 times.
 987747 const uint32_t i = order[at];
333
4/4
✓ Branch 0 taken 909616 times.
✓ Branch 1 taken 78131 times.
✓ Branch 2 taken 89324 times.
✓ Branch 3 taken 820292 times.
 987747 if (at + 1 < order.size() && staged.rows[order[at + 1]] == staged.rows[i]) {
334 89324 continue;
335 }
336
1/1
✓ Branch 1 taken 898423 times.
 898423 rows.push_back(staged.rows[i]);
337
1/1
✓ Branch 1 taken 898423 times.
 898423 const std::byte* cells = staged.bytes.data() + size_t(i) * cell;
338
1/1
✓ Branch 1 taken 898423 times.
 898423 bytes.insert(bytes.end(), cells, cells + cell);
339 }
340 78131 staged.rows = std::move(rows);
341 78131 staged.bytes = std::move(bytes);
342 78131 }
343
344 2 bool ColumnTable::Staged() const {
345
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 1 time.
 4 return std::ranges::any_of(m_staged, [](const Staging& s) { return !s.rows.empty(); });
346 }
347
348 15288126 void ColumnTable::Flush(const Scatter& scatter) {
349 15288126 constexpr uint32_t kScatterRuns = 16;
350 15288126 device::Queue& queue = m_heap.GetDevice().GetQueue();
351
2/2
✓ Branch 0 taken 67318243 times.
✓ Branch 1 taken 15288126 times.
 82606369 for (uint32_t column = 0; column < m_columns.size(); ++column) {
352
2/2
✓ Branch 0 taken 67240171 times.
✓ Branch 1 taken 78072 times.
 67318243 Staging& staged = m_staged[column];
353
2/2
✓ Branch 0 taken 67240171 times.
✓ Branch 1 taken 78072 times.
 67318243 if (staged.rows.empty()) {
354 67240174 continue;
355 }
356
4/5
✓ Branch 0 taken 6 times.
✓ Branch 1 taken 78066 times.
✓ Branch 3 taken 6 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 6 times.
 78072 CHRONO_BATCH_RENDER_ZONE("ColumnTable/Flush");
357
1/1
✓ Branch 1 taken 78072 times.
 78072 const uint32_t cell = m_columns[column].cellBytes;
358
1/1
✓ Branch 1 taken 78072 times.
 78072 Coalesce(staged, cell);
359 78072 const std::vector<uint32_t> rows = std::move(staged.rows);
360 78072 const std::vector<std::byte> cells = std::move(staged.bytes);
361 78072 staged = {};
362 78072 uint32_t runs = 0;
363
2/2
✓ Branch 0 taken 464171 times.
✓ Branch 1 taken 78072 times.
 542243 for (size_t at = 0; at < rows.size(); ++at) {
364
4/4
✓ Branch 0 taken 386099 times.
✓ Branch 1 taken 78072 times.
✓ Branch 2 taken 917 times.
✓ Branch 3 taken 385182 times.
 465088 runs += at == 0 || rows[at - 1] + 1 != rows[at] ? 1 : 0;
365 }
366
367
6/8
✓ Branch 0 taken 78026 times.
✓ Branch 1 taken 46 times.
✓ Branch 2 taken 3 times.
✓ Branch 3 taken 78023 times.
✓ Branch 4 taken 3 times.
✗ Branch 5 not taken.
✓ Branch 6 taken 3 times.
✗ Branch 7 not taken.
 78072 if (scatter && runs > kScatterRuns && cell % sizeof(uint32_t) == 0 &&
368
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 3 m_redirections[column].empty()) {
369
1/1
✓ Branch 1 taken 3 times.
 3 scatter(column, rows, cells);
370 3 continue;
371 }
372
2/2
✓ Branch 0 taken 78689 times.
✓ Branch 1 taken 78069 times.
 156758 for (uint32_t first = 0; first < rows.size();) {
373 uint32_t count = 1;
374
4/4
✓ Branch 0 taken 385802 times.
✓ Branch 1 taken 78069 times.
✓ Branch 2 taken 620 times.
✓ Branch 3 taken 385182 times.
 463871 while (first + count < rows.size() && rows[first + count] == rows[first] + count) {
375 385182 ++count;
376 }
377 78689 uint64_t at = uint64_t(first) * cell;
378
3/3
✓ Branch 1 taken 78689 times.
✓ Branch 3 taken 78695 times.
✓ Branch 4 taken 78689 times.
 157384 for (const device::BufferRange& range : Ranges(column, rows[first], count)) {
379
1/1
✓ Branch 1 taken 78695 times.
 78695 queue.Write(range, cells.data() + at);
380 78695 at += range.Bytes();
381 78689 }
382 78689 first += count;
383 }
384 78072 }
385 15288126 }
386
387 2290 std::vector<std::byte> ColumnTable::Read(uint32_t column, uint32_t first, uint32_t count) {
388
1/1
✓ Branch 1 taken 2290 times.
 2290 Flush();
389 2290 std::vector<std::byte> bytes(size_t(count) * CellBytes(column));
390 2290 uint64_t at = 0;
391
3/3
✓ Branch 1 taken 2290 times.
✓ Branch 3 taken 1179 times.
✓ Branch 4 taken 2290 times.
 3469 for (const device::BufferRange& range : Ranges(column, first, count)) {
392
1/1
✓ Branch 1 taken 1179 times.
 1179 m_heap.GetDevice().GetQueue().Read(range, bytes.data() + at);
393 1179 at += range.Bytes();
394 2290 }
395 2290 return bytes;
396 ✗ }
397
398 }
399