GCC Code Coverage Report

chrono_batch_render/
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
247 / 387
63.8%
Branches:
71 / 166
42.8%
Functions:
27 / 40
67.5%
device/vulkan/queue.cpp
Line Branch Exec Source
1 // SPDX-License-Identifier: BSD-3-Clause
2 #include <algorithm>
3 #include <chrono>
4 #include <cstdint>
5 #include <deque>
6 #include <map>
7 #include <memory>
8 #include <mutex>
9 #include <span>
10 #include <thread>
11 #include <utility>
12 #include <variant>
13 #include <vector>
14
15 #ifndef _WIN32
16 #include <unistd.h>
17 #endif
18
19 #include "chrono_batch_render/device/kernels/shared.h"
20 #include "chrono_batch_render/device/queue.h"
21 #include "chrono_batch_render/device/vulkan/device.h"
22 #include "chrono_batch_render/device/vulkan/interop.h"
23 #include "chrono_batch_render/device/vulkan/native.h"
24 #include "chrono_batch_render/util/error.h"
25 #include "chrono_batch_render/util/profiler.h"
26
27 namespace chrono::batch_render::device::vulkan {
28 namespace {
29
30 681117 void Barrier(const VolkDeviceTable& functions, VkCommandBuffer commands,
31 VkPipelineStageFlags2 stages, VkAccessFlags2 accesses) {
32 681117 VkMemoryBarrier2 barrier{VK_STRUCTURE_TYPE_MEMORY_BARRIER_2};
33 681117 barrier.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
34 681117 barrier.srcAccessMask = VK_ACCESS_2_MEMORY_WRITE_BIT;
35 681117 barrier.dstStageMask = stages;
36 681117 barrier.dstAccessMask = accesses;
37 681117 VkDependencyInfo dependency{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
38 681117 dependency.memoryBarrierCount = 1;
39 681117 dependency.pMemoryBarriers = &barrier;
40 681117 functions.vkCmdPipelineBarrier2(commands, &dependency);
41 681117 }
42
43 class VulkanQueue : public Queue {
44 public:
45 20 VulkanQueue(VulkanDevice& device, VkQueue handle, bool runsKernels)
46
2/2
✓ Branch 2 taken 20 times.
✓ Branch 5 taken 20 times.
 20 : Queue(device, runsKernels), m_vulkan(device), m_handle(handle) {
47
1/1
✓ Branch 1 taken 20 times.
 20 m_timeline = CreateTimeline();
48 20 VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO};
49 20 pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
50
1/1
✓ Branch 1 taken 20 times.
 20 pool.queueFamilyIndex = device.QueueFamily();
51 40 const VkResult created = Functions().vkCreateCommandPool(Handle(), &pool, nullptr, &m_pool);
52
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 if (created != VK_SUCCESS) {
53 ✗ Functions().vkDestroySemaphore(Handle(), m_timeline, nullptr);
54
0/1
✗ Branch 1 not taken.
 ✗ Check(created, "vkCreateCommandPool");
55 }
56 20 }
57
58 40 ~VulkanQueue() override {
59 20 Functions().vkQueueWaitIdle(m_handle);
60 20 Functions().vkDestroyCommandPool(Handle(), m_pool, nullptr);
61
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 m_interop.reset();
62
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 if (m_follow != VK_NULL_HANDLE) {
63 ✗ Functions().vkDestroySemaphore(Handle(), m_follow, nullptr);
64 }
65
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 20 times.
 20 for (const auto& [theirs, ours] : m_imported) {
66 ✗ Functions().vkDestroySemaphore(Handle(), ours, nullptr);
67 }
68 20 Functions().vkDestroySemaphore(Handle(), m_timeline, nullptr);
69 40 }
70
71 ✗ Operation SubmitNative(const NativeSubmit& submit) {
72 ✗ return RunNative(&submit, submit.zone);
73 }
74
75 ✗ VkResult Present(const VkPresentInfoKHR& present) {
76 ✗ const std::unique_lock submitLock = LockSubmissions();
77
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ return Functions().vkQueuePresentKHR(m_handle, &present);
78 ✗ }
79
80 ✗ VkQueue QueueHandle() const {
81 ✗ return m_handle;
82 }
83
84 protected:
85 80099 void Execute(const CommandBuffer& queued, const CommandBuffer& commands,
86 const Timestamps* timestamps, uint64_t value) override {
87 80099 const auto* queries = static_cast<const QueryTimestamps*>(timestamps);
88 80099 const VkCommandBuffer first = Begin(VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT);
89 80099 Translate(first, queued, nullptr);
90 80099 VkCommandBuffer second = VK_NULL_HANDLE;
91
2/2
✓ Branch 0 taken 80095 times.
✓ Branch 1 taken 4 times.
 80099 if (!commands.Reusable()) {
92 80095 Translate(first, commands, queries);
93 80095 End(first);
94 } else {
95 4 End(first);
96 4 second = RecordReusable(commands, queries);
97 }
98
99 80099 SubmitBuffers(first, second, value, nullptr);
100 80099 m_running.emplace_back(value, first);
101 80099 }
102
103 ✗ void ExecuteNative(const CommandBuffer& queued, uint64_t value, const void* native,
104 const Timestamps* timestamps) override {
105 ✗ const auto* queries = static_cast<const QueryTimestamps*>(timestamps);
106 ✗ const VolkDeviceTable& vk = Functions();
107 ✗ const VkCommandBuffer first = Begin(VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT);
108 ✗ Translate(first, queued, nullptr);
109 ✗ Barrier(vk, first, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
110 VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
111 ✗ VkCommandBuffer last = VK_NULL_HANDLE;
112
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (queries != nullptr) {
113 ✗ vk.vkCmdResetQueryPool(first, queries->pool, 0, queries->count);
114 ✗ vk.vkCmdWriteTimestamp2(first, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, queries->pool, 0);
115 ✗ last = Begin(VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT);
116 ✗ vk.vkCmdWriteTimestamp2(last, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, queries->pool, 1);
117 ✗ End(last);
118 }
119 ✗ End(first);
120 ✗ SubmitBuffers(first, VK_NULL_HANDLE, value, static_cast<const NativeSubmit*>(native), last);
121 ✗ m_running.emplace_back(value, first);
122
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (last != VK_NULL_HANDLE) {
123 ✗ m_running.emplace_back(value, last);
124 }
125 ✗ }
126
127 ✗ bool SharesStreams() override {
128 ✗ return Interop() != nullptr;
129 }
130
131 ✗ void FollowStream(void* stream) override {
132 ✗ Interop()->Follow(stream, ++m_followValue);
133 ✗ }
134
135 ✗ void StreamWait(void* stream, uint64_t value) override {
136 ✗ Interop()->Wait(stream, value);
137 ✗ }
138
139 ✗ bool FollowDevice(Queue& other, uint64_t value) override {
140 #ifdef _WIN32
141 (void)other;
142 (void)value;
143 return false;
144 #else
145 ✗ auto* theirs = dynamic_cast<VulkanQueue*>(&other);
146
0/6
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 ✗ if (theirs == nullptr || !m_vulkan.SharesSemaphores() ||
147
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ !theirs->m_vulkan.SharesSemaphores()) {
148 return false;
149 }
150 ✗ auto found = m_imported.find(theirs->m_timeline);
151
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (found == m_imported.end()) {
152 ✗ VkSemaphoreGetFdInfoKHR get{VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR};
153 ✗ get.semaphore = theirs->m_timeline;
154 ✗ get.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
155 ✗ int fd = -1;
156
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (theirs->Functions().vkGetSemaphoreFdKHR(theirs->Handle(), &get, &fd) !=
157 VK_SUCCESS) {
158 ✗ return false;
159 }
160 ✗ const VkSemaphore ours = CreateTimeline();
161 ✗ VkImportSemaphoreFdInfoKHR import{VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR};
162 ✗ import.semaphore = ours;
163 ✗ import.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
164 ✗ import.fd = fd;
165
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (Functions().vkImportSemaphoreFdKHR(Handle(), &import) != VK_SUCCESS) {
166 ✗ close(fd);
167 ✗ Functions().vkDestroySemaphore(Handle(), ours, nullptr);
168 ✗ return false;
169 }
170 ✗ found = m_imported.emplace(theirs->m_timeline, ours).first;
171 }
172 ✗ const VkSemaphore timeline = found->second;
173 ✗ const auto waiting = std::ranges::find(m_queueWaits, timeline, &QueueWait::timeline);
174
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (waiting != m_queueWaits.end()) {
175
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ waiting->value = std::max(waiting->value, value);
176 } else {
177 ✗ m_queueWaits.push_back({timeline, value});
178 }
179 return true;
180 #endif
181 }
182
183 ✗ void FollowQueue(Queue& other, uint64_t value) override {
184 ✗ const VkSemaphore timeline = static_cast<VulkanQueue&>(other).m_timeline;
185 ✗ const auto found = std::ranges::find(m_queueWaits, timeline, &QueueWait::timeline);
186
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (found != m_queueWaits.end()) {
187
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ found->value = std::max(found->value, value);
188 } else {
189 ✗ m_queueWaits.push_back({timeline, value});
190 }
191 ✗ }
192
193 11 std::shared_ptr<Timestamps> CreateTimestamps(uint32_t count) override {
194
2/4
✓ Branch 0 taken 11 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 11 times.
 22 return m_vulkan.HasTimestamps() ? std::make_shared<QueryTimestamps>(*this, count) : nullptr;
195 }
196
197 1161360 uint64_t Finished() const override {
198 1161360 uint64_t value = 0;
199 1161360 Check(Functions().vkGetSemaphoreCounterValue(Handle(), m_timeline, &value),
200 "vkGetSemaphoreCounterValue");
201 1161360 return value;
202 }
203
204 23218 void WaitFor(uint64_t value) const override {
205 23218 constexpr auto kSpin = std::chrono::microseconds(100);
206 23218 const auto until = std::chrono::steady_clock::now() + kSpin;
207 883996 do {
208
2/2
✓ Branch 1 taken 18218 times.
✓ Branch 2 taken 865778 times.
 883996 if (Finished() >= value) {
209 18218 return;
210 }
211 865778 std::this_thread::yield();
212
2/2
✓ Branch 1 taken 860778 times.
✓ Branch 2 taken 5000 times.
 865778 } while (std::chrono::steady_clock::now() < until);
213 5000 VkSemaphoreWaitInfo wait{VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO};
214 5000 wait.semaphoreCount = 1;
215 5000 wait.pSemaphores = &m_timeline;
216 5000 wait.pValues = &value;
217 5000 Check(Functions().vkWaitSemaphores(Handle(), &wait, UINT64_MAX), "vkWaitSemaphores");
218 }
219
220 private:
221 struct QueryTimestamps : Timestamps {
222
1/1
✓ Branch 1 taken 11 times.
 11 QueryTimestamps(const VulkanQueue& owner, uint32_t slots) : queue(owner), count(slots) {
223 11 VkQueryPoolCreateInfo create{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO};
224 11 create.queryType = VK_QUERY_TYPE_TIMESTAMP;
225 11 create.queryCount = slots;
226
1/1
✓ Branch 1 taken 11 times.
 11 Check(owner.Functions().vkCreateQueryPool(owner.Handle(), &create, nullptr, &pool),
227 "vkCreateQueryPool");
228 11 }
229
230 11 ~QueryTimestamps() override {
231 11 queue.Functions().vkDestroyQueryPool(queue.Handle(), pool, nullptr);
232 11 }
233
234 QueryTimestamps(const QueryTimestamps&) = delete;
235 QueryTimestamps& operator=(const QueryTimestamps&) = delete;
236
237 11 bool Read(std::span<uint64_t> ns) const override {
238 11 Check(queue.Functions().vkGetQueryPoolResults(queue.Handle(), pool, 0, count,
239 11 ns.size_bytes(), ns.data(),
240 sizeof(uint64_t), VK_QUERY_RESULT_64_BIT),
241 "vkGetQueryPoolResults");
242 11 return queue.m_vulkan.ToNanoseconds(ns);
243 }
244
245 const VulkanQueue& queue;
246 uint32_t count;
247 VkQueryPool pool = VK_NULL_HANDLE;
248 };
249
250 2481228 const VolkDeviceTable& Functions() const {
251
0/1
✗ Branch 3 not taken.
 31 return m_vulkan.Functions();
252 }
253
254 1166902 VkDevice Handle() const {
255
2/3
✓ Branch 3 taken 11 times.
✓ Branch 15 taken 20 times.
✗ Branch 18 not taken.
 1166453 return m_vulkan.Handle();
256 }
257
258 4 VkCommandBuffer RecordReusable(const CommandBuffer& commands, const QueryTimestamps* queries) {
259
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 2 times.
 4 std::shared_ptr<void>& recording = Recording(commands);
260
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 2 times.
 4 if (!recording) {
261 2 const VkCommandBuffer kept = Begin(0);
262 2 Translate(kept, commands, queries);
263 2 End(kept);
264 4 recording = std::shared_ptr<void>(kept, [this](void* handed) {
265 2 const std::lock_guard lock(m_returnedMutex);
266
1/1
✓ Branch 1 taken 2 times.
 2 m_returned.push_back(static_cast<VkCommandBuffer>(handed));
267
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 4 });
268 }
269 4 return static_cast<VkCommandBuffer>(recording.get());
270 }
271
272 80099 void SubmitBuffers(VkCommandBuffer first, VkCommandBuffer second, uint64_t value,
273 const NativeSubmit* native, VkCommandBuffer last = VK_NULL_HANDLE) {
274 80099 std::vector<VkCommandBufferSubmitInfo> buffers = {
275 80099 {VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, nullptr, first}};
276
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 80095 times.
 80099 if (second != VK_NULL_HANDLE) {
277
1/1
✓ Branch 1 taken 4 times.
 4 buffers.push_back({VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, nullptr, second});
278 }
279 80099 VkSemaphoreSubmitInfo signal{VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO};
280 80099 signal.semaphore = m_timeline;
281 80099 signal.value = value;
282 80099 signal.stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
283
1/1
✓ Branch 1 taken 80099 times.
 80099 std::vector<VkSemaphoreSubmitInfo> signals = {signal};
284 80099 std::vector<VkSemaphoreSubmitInfo> waits;
285
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 80099 times.
 80099 if (native != nullptr) {
286
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ for (const VkCommandBuffer commands : native->commands) {
287
0/1
✗ Branch 1 not taken.
 ✗ buffers.push_back(
288 {VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, nullptr, commands});
289 }
290
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (last != VK_NULL_HANDLE) {
291
0/1
✗ Branch 1 not taken.
 ✗ buffers.push_back({VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO, nullptr, last});
292 }
293
0/1
✗ Branch 1 not taken.
 ✗ waits.assign(native->waits.begin(), native->waits.end());
294
0/1
✗ Branch 1 not taken.
 ✗ signals.insert(signals.end(), native->signals.begin(), native->signals.end());
295 }
296
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 80099 times.
 80099 for (const QueueWait& queueWait : m_queueWaits) {
297
0/1
✗ Branch 1 not taken.
 ✗ VkSemaphoreSubmitInfo& wait = waits.emplace_back();
298 ✗ wait.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO;
299 ✗ wait.semaphore = queueWait.timeline;
300 ✗ wait.value = queueWait.value;
301 ✗ wait.stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
302 }
303 80099 m_queueWaits.clear();
304
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 80099 times.
 80099 if (m_followValue > m_followed) {
305
0/1
✗ Branch 1 not taken.
 ✗ VkSemaphoreSubmitInfo& wait = waits.emplace_back();
306 ✗ wait.sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO;
307 ✗ wait.semaphore = m_follow;
308 ✗ wait.value = m_followValue;
309 ✗ wait.stageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
310 ✗ m_followed = m_followValue;
311 }
312
313 80099 VkSubmitInfo2 submit{VK_STRUCTURE_TYPE_SUBMIT_INFO_2};
314
1/1
✓ Branch 1 taken 80099 times.
 80099 submit.commandBufferInfoCount = uint32_t(buffers.size());
315 80099 submit.pCommandBufferInfos = buffers.data();
316 80099 submit.signalSemaphoreInfoCount = uint32_t(signals.size());
317 80099 submit.pSignalSemaphoreInfos = signals.data();
318 80099 submit.waitSemaphoreInfoCount = uint32_t(waits.size());
319 80099 submit.pWaitSemaphoreInfos = waits.data();
320
2/2
✓ Branch 1 taken 80099 times.
✓ Branch 4 taken 80099 times.
 80099 Check(Functions().vkQueueSubmit2(m_handle, 1, &submit, VK_NULL_HANDLE), "vkQueueSubmit2");
321 80099 }
322
323 80101 VkCommandBuffer Begin(VkCommandBufferUsageFlags flags) {
324 80101 const VkCommandBuffer recorded = TakeCommandBuffer();
325 80101 VkCommandBufferBeginInfo begin{VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO};
326 80101 begin.flags = flags;
327 80101 Check(Functions().vkBeginCommandBuffer(recorded, &begin), "vkBeginCommandBuffer");
328 80101 return recorded;
329 }
330
331 80101 void End(VkCommandBuffer recorded) {
332 80101 Barrier(Functions(), recorded, VK_PIPELINE_STAGE_2_HOST_BIT, VK_ACCESS_2_HOST_READ_BIT);
333 80101 Check(Functions().vkEndCommandBuffer(recorded), "vkEndCommandBuffer");
334 80101 }
335
336 160196 void Translate(VkCommandBuffer recorded, const CommandBuffer& commands,
337 const QueryTimestamps* timestamps) {
338
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 160185 times.
 160196 const VolkDeviceTable& vk = Functions();
339
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 160185 times.
 160196 if (timestamps != nullptr) {
340 11 vk.vkCmdResetQueryPool(recorded, timestamps->pool, 0, timestamps->count);
341 }
342 160196 bool barrier = true;
343 160196 const std::span<const Command> list = commands.Commands();
344
2/2
✓ Branch 0 taken 1312795 times.
✓ Branch 1 taken 160196 times.
 1472991 for (size_t at = 0; at < list.size(); ++at) {
345
2/2
✓ Branch 0 taken 613411 times.
✓ Branch 1 taken 699384 times.
 1312795 const Command& command = list[at];
346
2/2
✓ Branch 0 taken 613411 times.
✓ Branch 1 taken 699384 times.
 1312795 if (std::holds_alternative<BarrierCommand>(command)) {
347 613411 barrier = true;
348 613411 continue;
349 }
350
2/2
✓ Branch 0 taken 601016 times.
✓ Branch 1 taken 98368 times.
 699384 if (const auto* zone = std::get_if<ZoneCommand>(&command)) {
351
2/2
✓ Branch 0 taken 36 times.
✓ Branch 1 taken 98332 times.
 98368 if (timestamps != nullptr) {
352 72 vk.vkCmdWriteTimestamp2(recorded, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
353 36 timestamps->pool,
354
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 18 times.
 36 2 * zone->index + (zone->begin ? 0 : 1));
355 }
356 98368 Label(recorded, commands, *zone);
357 98368 continue;
358 }
359
1/2
✓ Branch 0 taken 601016 times.
✗ Branch 1 not taken.
 601016 if (barrier) {
360 601016 Barrier(vk, recorded, VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
361 VK_ACCESS_2_MEMORY_READ_BIT | VK_ACCESS_2_MEMORY_WRITE_BIT);
362 barrier = false;
363 }
364
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 601016 times.
 601016 if (std::holds_alternative<BuildAccelCommand>(command)) {
365 ✗ size_t end = at + 1;
366
0/4
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ while (end < list.size() && std::holds_alternative<BuildAccelCommand>(list[end])) {
367 ✗ ++end;
368 }
369
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ RecordBuilds(recorded, list.subspan(at, end - at));
370 ✗ at = end - 1;
371 ✗ continue;
372 ✗ }
373 601016 Record(recorded, commands, command);
374 }
375 160196 }
376
377 ✗ void RecordBuilds(VkCommandBuffer recorded, std::span<const Command> commands) {
378
0/5
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 6 not taken.
 ✗ CHRONO_BATCH_RENDER_ZONE("Vulkan/RecordBuilds");
379
0/1
✗ Branch 1 not taken.
 ✗ std::vector<VulkanBuild> builds(commands.size());
380 ✗ std::vector<VkAccelerationStructureBuildGeometryInfoKHR> infos;
381 ✗ std::vector<VkAccelerationStructureBuildRangeInfoKHR> ranges;
382 ✗ std::vector<const VkAccelerationStructureBuildRangeInfoKHR*> pointers;
383
0/1
✗ Branch 1 not taken.
 ✗ infos.reserve(commands.size());
384
0/1
✗ Branch 1 not taken.
 ✗ ranges.reserve(commands.size());
385
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ for (size_t i = 0; i < commands.size(); ++i) {
386
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ const BuildAccelCommand& c = std::get<BuildAccelCommand>(commands[i]);
387
0/1
✗ Branch 1 not taken.
 ✗ VulkanBuild& build = builds[i];
388
0/1
✗ Branch 1 not taken.
 ✗ build.Describe(c.input);
389
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ build.info.dstAccelerationStructure = static_cast<const VulkanAccel&>(*c.dst).Handle();
390
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (c.update) {
391 ✗ build.info.mode = VK_BUILD_ACCELERATION_STRUCTURE_MODE_UPDATE_KHR;
392 ✗ build.info.srcAccelerationStructure = build.info.dstAccelerationStructure;
393 }
394
0/1
✗ Branch 1 not taken.
 ✗ build.info.scratchData.deviceAddress = c.scratch->Address() + c.scratchOffset;
395
0/1
✗ Branch 1 not taken.
 ✗ infos.push_back(build.info);
396
0/1
✗ Branch 1 not taken.
 ✗ ranges.push_back({build.primitives, 0, 0, 0});
397 }
398
399
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ for (const VkAccelerationStructureBuildRangeInfoKHR& range : ranges) {
400
0/1
✗ Branch 1 not taken.
 ✗ pointers.push_back(&range);
401 }
402
0/1
✗ Branch 1 not taken.
 ✗ Functions().vkCmdBuildAccelerationStructuresKHR(recorded, uint32_t(infos.size()),
403
0/1
✗ Branch 1 not taken.
 ✗ infos.data(), pointers.data());
404 ✗ }
405
406 98368 void Label(VkCommandBuffer recorded, const CommandBuffer& commands, const ZoneCommand& zone) {
407
1/2
✓ Branch 0 taken 98368 times.
✗ Branch 1 not taken.
 98368 if (!m_vulkan.Labels()) {
408 49184 return;
409 }
410
2/2
✓ Branch 0 taken 49184 times.
✓ Branch 1 taken 49184 times.
 98368 if (!zone.begin) {
411 49184 vkCmdEndDebugUtilsLabelEXT(recorded);
412 49184 return;
413 }
414 49184 VkDebugUtilsLabelEXT label{VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT};
415 49184 label.pLabelName = commands.Zones()[zone.index].Name().data();
416 49184 vkCmdBeginDebugUtilsLabelEXT(recorded, &label);
417 }
418
419 80101 VkCommandBuffer TakeCommandBuffer() {
420 80101 const uint64_t finished = Finished();
421
4/4
✓ Branch 0 taken 93573 times.
✓ Branch 1 taken 66531 times.
✓ Branch 2 taken 80003 times.
✓ Branch 3 taken 13570 times.
 160104 while (!m_running.empty() && m_running.front().first <= finished) {
422 80003 m_free.push_back(m_running.front().second);
423 80003 m_running.pop_front();
424 }
425 80101 {
426 80101 const std::lock_guard lock(m_returnedMutex);
427
1/1
✓ Branch 1 taken 80101 times.
 80101 m_free.insert(m_free.end(), m_returned.begin(), m_returned.end());
428 80101 m_returned.clear();
429 80101 }
430
431
2/2
✓ Branch 0 taken 79672 times.
✓ Branch 1 taken 429 times.
 80101 if (!m_free.empty()) {
432 79672 const VkCommandBuffer reused = m_free.back();
433 79672 m_free.pop_back();
434 79672 return reused;
435 }
436
437 429 VkCommandBufferAllocateInfo allocate{VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO};
438 429 allocate.commandPool = m_pool;
439 429 allocate.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
440 429 allocate.commandBufferCount = 1;
441 429 VkCommandBuffer allocated = VK_NULL_HANDLE;
442 429 Check(Functions().vkAllocateCommandBuffers(Handle(), &allocate, &allocated),
443 "vkAllocateCommandBuffers");
444 429 return allocated;
445 }
446
447 601016 void Record(VkCommandBuffer recorded, const CommandBuffer& commands, const Command& command) {
448 601016 const VolkDeviceTable& vk = Functions();
449 1202032 std::visit(
450 601016 Overloaded{
451 26589 [&](const CopyCommand& c) {
452 26589 const VkBufferCopy region{c.srcOffset, c.dstOffset, c.bytes};
453 26589 vk.vkCmdCopyBuffer(recorded, Native(*c.src), Native(*c.dst), 1, &region);
454 26589 },
455 130 [&](const TextureCopyCommand& c) { RecordTextureCopy(recorded, c); },
456 131696 [&](const FillCommand& c) {
457 131696 vk.vkCmdFillBuffer(recorded, Native(*c.dst), c.offset, c.bytes, c.value);
458 131696 },
459 210019 [&](const UpdateCommand& c) {
460 210019 vk.vkCmdUpdateBuffer(recorded, Native(*c.dst), c.offset, c.bytes,
461 210019 commands.Data(c));
462 210019 },
463 ✗ [&](const BuildAccelCommand&) { RecordBuilds(recorded, std::span(&command, 1)); },
464 [&](const BarrierCommand&) {},
465 [&](const ZoneCommand&) {},
466 232582 [&](const DispatchCommand& c) {
467 232582 Bind(recorded, *c.kernel, c.paramBytes, commands.Data(c));
468
2/2
✓ Branch 0 taken 77897 times.
✓ Branch 1 taken 154685 times.
 232582 if (c.counts != nullptr) {
469 77897 vk.vkCmdDispatchIndirect(recorded, Native(*c.counts), c.countsOffset);
470 } else {
471 154685 vk.vkCmdDispatch(recorded, c.groups.x, c.groups.y, c.groups.z);
472 }
473 232582 },
474 },
475 command);
476 601016 }
477
478 130 void RecordTextureCopy(VkCommandBuffer recorded, const TextureCopyCommand& c) {
479 130 const VolkDeviceTable& vk = Functions();
480 130 const VkImage image = static_cast<const VulkanTexture&>(*c.dst).Image();
481 130 const TextureDesc& desc = c.dst->Desc();
482 130 VkImageMemoryBarrier2 discard{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2};
483 130 discard.srcStageMask = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
484 130 discard.dstStageMask = VK_PIPELINE_STAGE_2_COPY_BIT;
485 130 discard.dstAccessMask = VK_ACCESS_2_TRANSFER_WRITE_BIT;
486 130 discard.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED;
487 130 discard.newLayout = VK_IMAGE_LAYOUT_GENERAL;
488 130 discard.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
489 130 discard.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
490 130 discard.image = image;
491 130 discard.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, c.level, 1, 0, 1};
492 130 VkDependencyInfo dependency{VK_STRUCTURE_TYPE_DEPENDENCY_INFO};
493 130 dependency.imageMemoryBarrierCount = 1;
494 130 dependency.pImageMemoryBarriers = &discard;
495 130 vk.vkCmdPipelineBarrier2(recorded, &dependency);
496
497 130 VkBufferImageCopy region{};
498 130 region.bufferOffset = c.srcOffset;
499 130 region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, c.level, 0, 1};
500
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 130 times.
 130 region.imageExtent = {std::max(desc.width >> c.level, 1u),
501
2/2
✓ Branch 0 taken 4 times.
✓ Branch 1 taken 126 times.
 130 std::max(desc.height >> c.level, 1u), 1};
502 130 vk.vkCmdCopyBufferToImage(recorded, Native(*c.src), image, VK_IMAGE_LAYOUT_GENERAL, 1,
503 &region);
504 130 }
505
506 232582 void Bind(VkCommandBuffer recorded, const Kernel& kernel, uint32_t paramBytes,
507 const void* params) {
508 232582 const VolkDeviceTable& vk = Functions();
509 232582 const VkPipelineLayout layout = m_vulkan.PipelineLayout();
510 232582 const VkDescriptorSet globals = m_vulkan.Globals();
511 232582 vk.vkCmdBindPipeline(recorded, VK_PIPELINE_BIND_POINT_COMPUTE,
512 static_cast<const VulkanKernel&>(kernel).Pipeline());
513 232582 vk.vkCmdBindDescriptorSets(recorded, VK_PIPELINE_BIND_POINT_COMPUTE, layout, 0, 1, &globals,
514 0, nullptr);
515
1/2
✓ Branch 0 taken 232582 times.
✗ Branch 1 not taken.
 232582 if (paramBytes > 0) {
516 232582 vk.vkCmdPushConstants(recorded, layout, VK_SHADER_STAGE_COMPUTE_BIT, 0,
517 232582 uint32_t(RoundUp(paramBytes, 4)), params);
518 }
519 232582 }
520
521 446331 static VkBuffer Native(Buffer& buffer) {
522 446331 return static_cast<VulkanBuffer&>(buffer).Handle();
523 }
524
525 20 VkSemaphore CreateTimeline() {
526 20 VkSemaphoreTypeCreateInfo type{VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO};
527 20 type.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE;
528 20 VkExportSemaphoreCreateInfo exported{VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO};
529 20 exported.handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
530
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 if (m_vulkan.SharesSemaphores()) {
531 ✗ type.pNext = &exported;
532 }
533 20 VkSemaphoreCreateInfo semaphore{VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO, &type};
534 20 VkSemaphore created = VK_NULL_HANDLE;
535 20 Check(Functions().vkCreateSemaphore(Handle(), &semaphore, nullptr, &created),
536 "vkCreateSemaphore");
537 20 return created;
538 }
539
540 ✗ StreamInterop* Interop() {
541 ✗ const std::lock_guard lock(m_interopMutex);
542
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (!m_interopTried) {
543 ✗ m_interopTried = true;
544
0/1
✗ Branch 1 not taken.
 ✗ m_follow = CreateTimeline();
545 ✗ try {
546
0/1
✗ Branch 1 not taken.
 ✗ m_interop = CreateStreamInterop(m_vulkan, m_timeline, m_follow);
547
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ } catch (const UnsupportedError&) {
548 ✗ }
549 }
550 ✗ return m_interop.get();
551 ✗ }
552
553 struct QueueWait {
554 VkSemaphore timeline;
555 uint64_t value;
556 };
557
558 std::map<VkSemaphore, VkSemaphore> m_imported;
559
560 VulkanDevice& m_vulkan;
561 VkQueue m_handle;
562 std::vector<QueueWait> m_queueWaits;
563 VkSemaphore m_timeline = VK_NULL_HANDLE;
564 VkSemaphore m_follow = VK_NULL_HANDLE;
565 uint64_t m_followValue = 0;
566 uint64_t m_followed = 0;
567 std::mutex m_interopMutex;
568 bool m_interopTried = false;
569 std::unique_ptr<StreamInterop> m_interop;
570 VkCommandPool m_pool = VK_NULL_HANDLE;
571 std::deque<std::pair<uint64_t, VkCommandBuffer>> m_running;
572 std::vector<VkCommandBuffer> m_free;
573 std::mutex m_returnedMutex;
574 std::vector<VkCommandBuffer> m_returned;
575 };
576
577 }
578
579 20 std::unique_ptr<Queue> CreateQueue(VulkanDevice& device, VkQueue handle, bool runsKernels) {
580 20 return std::make_unique<VulkanQueue>(device, handle, runsKernels);
581 }
582
583 ✗ VkQueue NativeQueue(Queue& queue) {
584 ✗ return static_cast<VulkanQueue&>(queue).QueueHandle();
585 }
586
587 ✗ Operation SubmitNative(Queue& queue, const NativeSubmit& submit) {
588 ✗ return static_cast<VulkanQueue&>(queue).SubmitNative(submit);
589 }
590
591 ✗ VkResult PresentNative(Queue& queue, const VkPresentInfoKHR& present) {
592 ✗ return static_cast<VulkanQueue&>(queue).Present(present);
593 }
594
595 }
596