GCC Code Coverage Report

chrono_batch_render/
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
137 / 165
83.0%
Branches:
95 / 140
67.9%
Functions:
16 / 17
94.1%
stage/columns/array_pool.cpp
Line Branch Exec Source
1 // SPDX-License-Identifier: BSD-3-Clause
2 #include "chrono_batch_render/stage/columns/array_pool.h"
3
4 #include <algorithm>
5 #include <numeric>
6 #include <utility>
7
8 #include "chrono_batch_render/device/kernels/shared.h"
9 #include "chrono_batch_render/device/queue.h"
10 #include "chrono_batch_render/util/error.h"
11
12 namespace chrono::batch_render {
13
14 namespace {
15
16 constexpr uint64_t kArrayAlignment = 16;
17
18 }
19
20 563 ArrayPool::ArrayPool(DeviceHeap& heap, std::string label, uint32_t itemBytes)
21
1/2
✓ Branch 1 taken 563 times.
✗ Branch 2 not taken.
 563 : m_heap(heap), m_label(std::move(label)), m_itemBytes(itemBytes) {
22
2/4
✓ Branch 0 taken 563 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 563 times.
 563 if (itemBytes == 0 || itemBytes % 4 != 0) {
23
0/1
✗ Branch 2 not taken.
 ✗ throw Error("{} holds items of {} bytes; items are whole words", m_label, itemBytes);
24 }
25 563 }
26
27 3053 uint32_t ArrayPool::AddRun(HeapBlock block, uint32_t live) {
28
2/2
✓ Branch 0 taken 201 times.
✓ Branch 1 taken 2852 times.
 3053 if (!m_freeRuns.empty()) {
29 201 const uint32_t run = m_freeRuns.back();
30 201 m_freeRuns.pop_back();
31
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 201 times.
 201 m_runs[run] = {block, live, {}};
32 201 return run;
33 }
34
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2852 times.
 2852 m_runs.push_back({block, live, {}});
35 2852 return uint32_t(m_runs.size() - 1);
36 3053 }
37
38 3099 ArrayRef ArrayPool::Allocate(uint32_t count, uint32_t capacity) {
39
2/2
✓ Branch 0 taken 3044 times.
✓ Branch 1 taken 55 times.
 3099 capacity = std::max(capacity, count);
40
2/2
✓ Branch 0 taken 55 times.
✓ Branch 1 taken 3044 times.
 3099 if (capacity == 0) {
41 55 return {};
42 }
43 3044 const uint64_t bytes = RoundUp(uint64_t(capacity) * m_itemBytes, kArrayAlignment);
44 3044 const HeapBlock block = m_heap.Allocate(bytes, kArrayAlignment);
45 3044 const uint32_t run = AddRun(block, 1);
46 3044 m_arrays.emplace(block.Address(), Entry{run, 1});
47 3044 return {block.Address(), count, capacity};
48 }
49
50 9 std::vector<ArrayRef> ArrayPool::AllocateRun(std::span<const uint32_t> counts) {
51 9 std::vector<ArrayRef> refs(counts.size());
52 9 uint64_t items = 0;
53 9 uint32_t live = 0;
54
2/2
✓ Branch 0 taken 25 times.
✓ Branch 1 taken 9 times.
 34 for (const uint32_t count : counts) {
55 25 items += count;
56
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 23 times.
 27 live += count > 0 ? 1 : 0;
57 }
58
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 if (items == 0) {
59 return refs;
60 }
61 9 const uint64_t unit = std::lcm(uint64_t(m_itemBytes), kArrayAlignment);
62
1/1
✓ Branch 1 taken 9 times.
 9 const HeapBlock block = m_heap.Allocate(RoundUp(items * m_itemBytes, unit), unit);
63
1/1
✓ Branch 1 taken 9 times.
 9 const uint32_t run = AddRun(block, live);
64 9 uint64_t address = block.Address();
65
2/2
✓ Branch 0 taken 25 times.
✓ Branch 1 taken 9 times.
 34 for (uint32_t i = 0; i < counts.size(); ++i) {
66
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 23 times.
 25 if (counts[i] == 0) {
67 2 continue;
68 }
69
1/1
✓ Branch 1 taken 23 times.
 23 m_arrays.emplace(address, Entry{run, 1});
70 23 refs[i] = {address, counts[i], counts[i]};
71 23 address += uint64_t(counts[i]) * m_itemBytes;
72 }
73 return refs;
74 ✗ }
75
76 ✗ std::vector<ArrayRef> ArrayPool::Borrow(Ref<device::Buffer> buffer, uint32_t arrays,
77 uint32_t count) {
78 ✗ const uint64_t stride = uint64_t(count) * m_itemBytes;
79
0/6
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 ✗ if (arrays == 0 || count == 0 || buffer->Bytes() < stride * arrays) {
80
0/1
✗ Branch 1 not taken.
 ✗ throw Error("{} borrows {} arrays of {} items from {} bytes", m_label, arrays, count,
81
0/1
✗ Branch 2 not taken.
 ✗ buffer->Bytes());
82 }
83 ✗ const uint32_t run = AddRun({buffer.Get(), 0, buffer->Bytes()}, arrays);
84
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ m_runs[run].borrowed = std::move(buffer);
85 ✗ std::vector<ArrayRef> refs;
86
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ for (uint32_t i = 0; i < arrays; ++i) {
87
0/1
✗ Branch 1 not taken.
 ✗ const uint64_t address = m_runs[run].block.Address() + i * stride;
88
0/3
✗ Branch 1 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 ✗ if (!m_arrays.emplace(address, Entry{run, 1}).second) {
89
0/1
✗ Branch 2 not taken.
 ✗ throw Error("{} already holds an array at {:#x}", m_label, address);
90 }
91
0/1
✗ Branch 1 not taken.
 ✗ refs.push_back({address, count, count});
92 }
93 ✗ return refs;
94 ✗ }
95
96 201 bool ArrayPool::Borrowed(const ArrayRef& array) const {
97
2/4
✓ Branch 0 taken 201 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 201 times.
✗ Branch 4 not taken.
 201 return array.address != 0 && m_runs[Lookup(array).run].borrowed;
98 }
99
100 6293 const ArrayPool::Entry& ArrayPool::Lookup(const ArrayRef& array) const {
101 6293 const auto found = m_arrays.find(array.address);
102
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 6292 times.
 6293 if (found == m_arrays.end()) {
103
1/1
✓ Branch 2 taken 1 time.
 1 throw Error("{} holds no array at {:#x}", m_label, array.address);
104 }
105 6292 return found->second;
106 }
107
108 164 ArrayRef ArrayPool::Share(const ArrayRef& array) {
109
1/2
✓ Branch 0 taken 164 times.
✗ Branch 1 not taken.
 164 if (array.address != 0) {
110 164 Lookup(array);
111 164 ++m_arrays.at(array.address).references;
112 }
113 164 return array;
114 }
115
116 116 uint32_t ArrayPool::References(const ArrayRef& array) const {
117
1/2
✓ Branch 0 taken 116 times.
✗ Branch 1 not taken.
 116 return array.address == 0 ? 0 : Lookup(array).references;
118 }
119
120 466 void ArrayPool::Free(const ArrayRef& array) {
121
2/2
✓ Branch 0 taken 382 times.
✓ Branch 1 taken 84 times.
 466 if (array.address == 0) {
122 return;
123 }
124 382 Lookup(array);
125 382 Entry& entry = m_arrays.at(array.address);
126
2/2
✓ Branch 0 taken 347 times.
✓ Branch 1 taken 35 times.
 382 if (--entry.references > 0) {
127 return;
128 }
129
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 347 times.
 347 if (const auto staged = m_stagedIndices.find(array.address); staged != m_stagedIndices.end()) {
130 ✗ m_staged[staged->second].bytes = 0;
131 ✗ m_stagedIndices.erase(staged);
132 }
133
2/2
✓ Branch 0 taken 343 times.
✓ Branch 1 taken 4 times.
 347 Run& run = m_runs[entry.run];
134
2/2
✓ Branch 0 taken 343 times.
✓ Branch 1 taken 4 times.
 347 if (--run.live == 0) {
135
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 343 times.
 343 if (run.borrowed) {
136
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ run.borrowed = {};
137 } else {
138 343 m_heap.Free(run.block);
139 }
140 343 m_freeRuns.push_back(entry.run);
141 }
142 347 m_arrays.erase(array.address);
143 }
144
145 4218 device::BufferRange ArrayPool::Range(const ArrayRef& array) const {
146 4218 const Run& run = m_runs[Lookup(array).run];
147 4217 return run.block.buffer->Slice(array.address - run.block.buffer->Address(),
148 4217 uint64_t(array.capacity) * m_itemBytes);
149 }
150
151 1656 void ArrayPool::Write(const ArrayRef& array, std::span<const std::byte> bytes) {
152
2/2
✓ Branch 0 taken 1642 times.
✓ Branch 1 taken 14 times.
 1656 if (bytes.empty()) {
153 return;
154 }
155
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1642 times.
 1642 if (bytes.size() > uint64_t(array.capacity) * m_itemBytes) {
156 ✗ throw Error("{} bytes for an array of {} {}-byte items", bytes.size(), array.capacity,
157
0/1
✗ Branch 2 not taken.
 ✗ m_itemBytes);
158 }
159 1642 m_heap.GetDevice().GetQueue().Write(Range(array).Slice(0, bytes.size()), bytes.data());
160 }
161
162 732 void ArrayPool::Stage(const ArrayRef& array, std::span<const std::byte> bytes) {
163
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 732 times.
 732 if (bytes.size() % sizeof(uint32_t) != 0) {
164 ✗ Write(array, bytes);
165 ✗ return;
166 }
167
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 732 times.
 732 if (bytes.size() > uint64_t(array.capacity) * m_itemBytes) {
168 ✗ throw Error("{} bytes for an array of {} {}-byte items", bytes.size(), array.capacity,
169
0/1
✗ Branch 2 not taken.
 ✗ m_itemBytes);
170 }
171
2/2
✓ Branch 1 taken 17 times.
✓ Branch 2 taken 715 times.
 732 if (const auto staged = m_stagedIndices.find(array.address); staged != m_stagedIndices.end()) {
172 17 m_staged[staged->second].bytes = 0;
173 }
174 732 m_stagedIndices[array.address] = m_staged.size();
175 732 m_staged.push_back({array, m_stagedBytes.size(), bytes.size()});
176 732 m_stagedBytes.insert(m_stagedBytes.end(), bytes.begin(), bytes.end());
177 }
178
179 376 std::vector<ArrayPool::StagedCopy> ArrayPool::TakeStaged(uint64_t base) {
180 376 std::vector<StagedCopy> copies;
181
2/2
✓ Branch 0 taken 693 times.
✓ Branch 1 taken 376 times.
 1069 for (const Staged& staged : m_staged) {
182
2/2
✓ Branch 0 taken 685 times.
✓ Branch 1 taken 8 times.
 693 if (staged.bytes > 0) {
183
1/1
✓ Branch 1 taken 685 times.
 1370 copies.push_back({Range(staged.array).Address(), base + staged.offset,
184
1/1
✓ Branch 1 taken 685 times.
 685 uint32_t(staged.bytes / sizeof(uint32_t))});
185 }
186 }
187 376 m_staged.clear();
188 376 m_stagedBytes.clear();
189 376 m_stagedIndices.clear();
190 376 return copies;
191 ✗ }
192
193 3114 void ArrayPool::FlushStaged() {
194
2/2
✓ Branch 0 taken 22 times.
✓ Branch 1 taken 3114 times.
 3136 for (const Staged& staged : m_staged) {
195
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 22 times.
 22 if (staged.bytes > 0) {
196
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ Write(staged.array, std::span(m_stagedBytes).subspan(staged.offset, staged.bytes));
197 }
198 }
199 3114 m_staged.clear();
200 3114 m_stagedBytes.clear();
201 3114 m_stagedIndices.clear();
202 3114 }
203
204 1536 std::vector<std::byte> ArrayPool::Read(const ArrayRef& array) {
205 1536 FlushStaged();
206 1536 std::vector<std::byte> bytes(size_t(array.count) * m_itemBytes);
207
1/2
✓ Branch 0 taken 1536 times.
✗ Branch 1 not taken.
 1536 if (!bytes.empty()) {
208
3/3
✓ Branch 1 taken 1535 times.
✓ Branch 4 taken 1535 times.
✓ Branch 7 taken 1535 times.
 1536 m_heap.GetDevice().GetQueue().Read(Range(array).Slice(0, bytes.size()), bytes.data());
209 }
210 1535 return bytes;
211 1 }
212
213 1578 std::vector<std::vector<std::byte>> ArrayPool::Read(std::span<const ArrayRef> arrays) {
214 1578 FlushStaged();
215
216 500 struct Extent {
217 device::Buffer* buffer = nullptr;
218 uint64_t first = ~uint64_t(0);
219 uint64_t last = 0;
220 device::Download download;
221 };
222
223 1578 std::vector<Extent> spans;
224
1/1
✓ Branch 1 taken 1578 times.
 1578 std::vector<std::pair<size_t, uint64_t>> located(arrays.size(), {0, 0});
225
2/2
✓ Branch 0 taken 1731 times.
✓ Branch 1 taken 1578 times.
 3309 for (size_t i = 0; i < arrays.size(); ++i) {
226
3/4
✓ Branch 0 taken 1212 times.
✓ Branch 1 taken 519 times.
✗ Branch 2 not taken.
✓ Branch 3 taken 1212 times.
 1731 if (arrays[i].address == 0 || arrays[i].count == 0) {
227 519 continue;
228 }
229
1/1
✓ Branch 1 taken 1212 times.
 1212 device::Buffer* buffer = m_runs[Lookup(arrays[i]).run].block.buffer;
230 1212 auto found = std::ranges::find(spans, buffer, &Extent::buffer);
231
2/2
✓ Branch 0 taken 250 times.
✓ Branch 1 taken 962 times.
 1212 if (found == spans.end()) {
232
2/2
✓ Branch 1 taken 250 times.
✓ Branch 4 taken 250 times.
 250 found = spans.insert(spans.end(), Extent{.buffer = buffer});
233 }
234
2/2
✓ Branch 0 taken 381 times.
✓ Branch 1 taken 831 times.
 1212 const uint64_t offset = arrays[i].address - buffer->Address();
235
2/2
✓ Branch 0 taken 381 times.
✓ Branch 1 taken 831 times.
 1212 found->first = std::min(found->first, offset);
236
2/2
✓ Branch 0 taken 465 times.
✓ Branch 1 taken 747 times.
 1212 found->last = std::max(found->last, offset + uint64_t(arrays[i].count) * m_itemBytes);
237 1212 located[i] = {size_t(found - spans.begin()), offset};
238 }
239 1578 device::Queue& queue = m_heap.GetDevice().GetQueue();
240
2/2
✓ Branch 0 taken 250 times.
✓ Branch 1 taken 1578 times.
 1828 for (Extent& span : spans) {
241
2/2
✓ Branch 1 taken 250 times.
✓ Branch 4 taken 250 times.
 250 span.download = queue.ReadAsync(span.buffer->Slice(span.first, span.last - span.first));
242 }
243
2/2
✓ Branch 0 taken 250 times.
✓ Branch 1 taken 1578 times.
 1828 for (Extent& span : spans) {
244
1/1
✓ Branch 1 taken 250 times.
 250 span.download.Wait();
245 }
246
1/1
✓ Branch 1 taken 1578 times.
 1578 std::vector<std::vector<std::byte>> bytes(arrays.size());
247
2/2
✓ Branch 0 taken 1731 times.
✓ Branch 1 taken 1578 times.
 3309 for (size_t i = 0; i < arrays.size(); ++i) {
248
3/4
✓ Branch 0 taken 1212 times.
✓ Branch 1 taken 519 times.
✓ Branch 2 taken 1212 times.
✗ Branch 3 not taken.
 1731 if (arrays[i].address == 0 || arrays[i].count == 0) {
249 519 continue;
250 }
251
1/1
✓ Branch 1 taken 1212 times.
 1212 const Extent& span = spans[located[i].first];
252
2/2
✓ Branch 1 taken 1212 times.
✓ Branch 4 taken 1212 times.
 2424 const std::span<const std::byte> read = span.download.Bytes().subspan(
253
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1212 times.
 1212 located[i].second - span.first, uint64_t(arrays[i].count) * m_itemBytes);
254
1/1
✓ Branch 1 taken 1212 times.
 1212 bytes[i].assign(read.begin(), read.end());
255 }
256 3156 return bytes;
257 1578 }
258
259 }
260