GCC Code Coverage Report

chrono_batch_render/
Coverage:
low: ≥ 0%
medium: ≥ 75.0%
high: ≥ 90.0%
Lines:
570 / 875
65.1%
Branches:
218 / 478
45.6%
Functions:
47 / 68
69.1%
device/vulkan/device.cpp
Line Branch Exec Source
1 // SPDX-License-Identifier: BSD-3-Clause
2 #ifndef _WIN32
3 #include <unistd.h>
4 #endif
5
6 #include "chrono_batch_render/device/vulkan/device.h"
7
8 #include <algorithm>
9 #include <cstring>
10 #include <optional>
11 #include <string>
12 #include <string_view>
13 #include <utility>
14 #include <variant>
15 #include <vector>
16
17 #ifdef _WIN32
18 #include <windows.h>
19 #endif
20
21 #include <fmt/format.h>
22
23 #include "chrono_batch_render/device/commands.h"
24 #include "chrono_batch_render/device/queue.h"
25 #include "chrono_batch_render/device/slang/compiler.h"
26 #include "chrono_batch_render/device/vulkan/interop.h"
27 #include "chrono_batch_render/device/vulkan/native.h"
28 #include "chrono_batch_render/util/disk_cache.h"
29 #include "chrono_batch_render/util/error.h"
30 #include "chrono_batch_render/util/log.h"
31 #include "chrono_batch_render/util/text.h"
32
33 namespace chrono::batch_render::device::vulkan {
34
35 21 std::array<uint8_t, 16> DeviceUuid(VkPhysicalDevice device) {
36 21 VkPhysicalDeviceIDProperties id{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES};
37 21 VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &id};
38 21 vkGetPhysicalDeviceProperties2(device, &properties);
39 21 std::array<uint8_t, 16> uuid{};
40 21 std::copy(std::begin(id.deviceUUID), std::end(id.deviceUUID), uuid.begin());
41 21 return uuid;
42 }
43
44 namespace {}
45
46 104 std::optional<ComputeFamily> ComputeQueueFamily(VkPhysicalDevice device, VkQueueFlags flags) {
47 104 const std::vector<VkQueueFamilyProperties> families = Enumerate<VkQueueFamilyProperties>(
48 208 [device](uint32_t* count, VkQueueFamilyProperties* items) {
49
1/1
✓ Branch 2 taken 104 times.
 208 vkGetPhysicalDeviceQueueFamilyProperties(device, count, items);
50 208 });
51
1/2
✓ Branch 0 taken 104 times.
✗ Branch 1 not taken.
 104 for (uint32_t family = 0; family < uint32_t(families.size()); ++family) {
52
1/2
✓ Branch 0 taken 104 times.
✗ Branch 1 not taken.
 104 if ((families[family].queueFlags & flags) == flags) {
53 104 return ComputeFamily{family, families[family]};
54 }
55 }
56 ✗ return std::nullopt;
57 104 }
58
59 namespace {
60
61 #if defined(_WIN32)
62 constexpr VkTimeDomainKHR kHostTimeDomain = VK_TIME_DOMAIN_QUERY_PERFORMANCE_COUNTER_KHR;
63
64 uint64_t HostClockNs(uint64_t ticks) {
65 static const uint64_t frequency = [] {
66 LARGE_INTEGER value{};
67 QueryPerformanceFrequency(&value);
68 return uint64_t(value.QuadPart);
69 }();
70 return ticks / frequency * 1'000'000'000 + ticks % frequency * 1'000'000'000 / frequency;
71 }
72 #else
73 constexpr VkTimeDomainKHR kHostTimeDomain = VK_TIME_DOMAIN_CLOCK_MONOTONIC_KHR;
74
75 11 uint64_t HostClockNs(uint64_t ticks) {
76 11 return ticks;
77 }
78 #endif
79
80 constexpr VkBufferUsageFlags kStorageUsage =
81 VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
82 VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT |
83 VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT;
84
85 20 VkPhysicalDevice FindPhysicalDevice(VkInstance instance, const AdapterInfo& adapter) {
86
1/2
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
 20 for (VkPhysicalDevice device : PhysicalDevices(instance)) {
87
2/3
✓ Branch 1 taken 20 times.
✓ Branch 3 taken 20 times.
✗ Branch 4 not taken.
 20 if (DeviceUuid(device) == adapter.uuid) {
88 20 return device;
89 }
90 20 }
91
0/1
✗ Branch 2 not taken.
 ✗ throw Error("the Vulkan adapter {} is no longer present", adapter.name);
92 }
93
94 #if !defined(_WIN32)
95 20 bool ExportsBuffers(VkPhysicalDevice device) {
96 20 VkPhysicalDeviceExternalBufferInfo info{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_BUFFER_INFO};
97 20 info.usage = kStorageUsage;
98 20 info.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
99 20 VkExternalBufferProperties properties{VK_STRUCTURE_TYPE_EXTERNAL_BUFFER_PROPERTIES};
100 20 vkGetPhysicalDeviceExternalBufferProperties(device, &info, &properties);
101 20 return (properties.externalMemoryProperties.externalMemoryFeatures &
102 20 VK_EXTERNAL_MEMORY_FEATURE_EXPORTABLE_BIT) != 0;
103 }
104
105 ✗ bool ExportsTimelines(VkPhysicalDevice device) {
106 ✗ VkSemaphoreTypeCreateInfo timeline{VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO};
107 ✗ timeline.semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE;
108 ✗ VkPhysicalDeviceExternalSemaphoreInfo info{
109 ✗ VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_SEMAPHORE_INFO, &timeline};
110 ✗ info.handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT;
111 ✗ VkExternalSemaphoreProperties properties{VK_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_PROPERTIES};
112 ✗ vkGetPhysicalDeviceExternalSemaphoreProperties(device, &info, &properties);
113 ✗ return (properties.externalSemaphoreFeatures & VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT) !=
114 ✗ 0;
115 }
116 #endif
117
118 20 const char* CalibratedTimestampsExtension(VkPhysicalDevice device, const Extensions& extensions) {
119 #if defined(__APPLE__)
120 return nullptr;
121 #endif
122 20 const bool khr = extensions.Has(VK_KHR_CALIBRATED_TIMESTAMPS_EXTENSION_NAME);
123
2/4
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 20 times.
✗ Branch 4 not taken.
 20 if (!khr && !extensions.Has(VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME)) {
124 return nullptr;
125 }
126
127
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 const auto list = khr ? vkGetPhysicalDeviceCalibrateableTimeDomainsKHR
128 : vkGetPhysicalDeviceCalibrateableTimeDomainsEXT;
129 20 const std::vector<VkTimeDomainKHR> domains = Enumerate<VkTimeDomainKHR>(
130
1/1
✓ Branch 2 taken 20 times.
 60 [&](uint32_t* count, VkTimeDomainKHR* items) { list(device, count, items); });
131 60 const auto has = [&](VkTimeDomainKHR domain) {
132 40 return std::ranges::find(domains, domain) != domains.end();
133 20 };
134
2/4
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
✓ Branch 4 taken 20 times.
✗ Branch 5 not taken.
 20 if (!has(VK_TIME_DOMAIN_DEVICE_KHR) || !has(kHostTimeDomain)) {
135 return nullptr;
136 }
137 20 return khr ? VK_KHR_CALIBRATED_TIMESTAMPS_EXTENSION_NAME
138
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 : VK_EXT_CALIBRATED_TIMESTAMPS_EXTENSION_NAME;
139 20 }
140
141 ✗ struct DeviceExtensions {
142 std::vector<const char*> names;
143 const char* calibrated = nullptr;
144 bool sharesSemaphores = false;
145 bool sharesMemory = false;
146 bool capturesStatistics = false;
147 bool presentation = false;
148 };
149
150 20 DeviceExtensions ChooseExtensions(VkPhysicalDevice device, const AdapterInfo& adapter,
151 const DeviceDesc& desc, bool hardware) {
152 20 DeviceExtensions chosen;
153
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 if (hardware) {
154
0/1
✗ Branch 1 not taken.
 ✗ chosen.names.assign(std::begin(kRayQueryExtensions), std::end(kRayQueryExtensions));
155 }
156
1/1
✓ Branch 1 taken 20 times.
 20 const Extensions available(device);
157
2/3
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 20 times.
 20 chosen.calibrated = desc.profiling ? CalibratedTimestampsExtension(device, available) : nullptr;
158
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 if (chosen.calibrated != nullptr) {
159
1/1
✓ Branch 1 taken 20 times.
 20 chosen.names.push_back(chosen.calibrated);
160 }
161
2/3
✓ Branch 1 taken 20 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 20 times.
 20 if (available.Has("VK_KHR_portability_subset")) {
162
0/1
✗ Branch 1 not taken.
 ✗ chosen.names.push_back("VK_KHR_portability_subset");
163 }
164 #if !defined(_WIN32)
165 40 chosen.sharesSemaphores =
166
2/6
✓ Branch 1 taken 20 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 20 times.
✗ Branch 6 not taken.
✗ Branch 8 not taken.
✗ Branch 9 not taken.
 20 available.Has(VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME) && ExportsTimelines(device);
167
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 if (chosen.sharesSemaphores) {
168
0/1
✗ Branch 1 not taken.
 ✗ chosen.names.push_back(VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME);
169 }
170 40 chosen.sharesMemory =
171
4/6
✓ Branch 1 taken 20 times.
✓ Branch 3 taken 20 times.
✗ Branch 4 not taken.
✓ Branch 6 taken 20 times.
✗ Branch 8 not taken.
✓ Branch 9 taken 20 times.
 20 available.Has(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME) && ExportsBuffers(device);
172
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 if (chosen.sharesMemory) {
173
1/1
✓ Branch 1 taken 20 times.
 20 chosen.names.push_back(VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME);
174 }
175 #endif
176 20 chosen.presentation = desc.presentation;
177
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 if (chosen.presentation) {
178
0/1
✗ Branch 1 not taken.
 ✗ chosen.names.push_back(VK_KHR_SWAPCHAIN_EXTENSION_NAME);
179 }
180
2/4
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
✓ Branch 2 taken 20 times.
✗ Branch 3 not taken.
 20 chosen.capturesStatistics = desc.profiling && adapter.features.kernelStatistics;
181
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 if (chosen.capturesStatistics) {
182
0/1
✗ Branch 1 not taken.
 ✗ chosen.names.push_back(VK_KHR_PIPELINE_EXECUTABLE_PROPERTIES_EXTENSION_NAME);
183 }
184 20 return chosen;
185 20 }
186
187 20 VkDevice CreateLogicalDevice(VkPhysicalDevice physicalDevice, uint32_t queueFamily,
188 uint32_t queueCount, const DeviceExtensions& extensions, bool hardware,
189 bool textures, bool blockCompression, bool anisotropy,
190 const Creator* creator) {
191 20 const std::vector<const char*>& names = extensions.names;
192 20 FeatureChain enabled;
193 20 enabled.Link([&](const char* name) {
194 80 return std::ranges::any_of(
195
1/2
✓ Branch 0 taken 160 times.
✗ Branch 1 not taken.
 160 names, [name](const char* each) { return std::strcmp(each, name) == 0; });
196 });
197
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 enabled.executables.pipelineExecutableInfo = extensions.capturesStatistics ? VK_TRUE : VK_FALSE;
198 20 enabled.base.features.shaderInt64 = VK_TRUE;
199 20 enabled.vulkan12.bufferDeviceAddress = VK_TRUE;
200 20 enabled.vulkan12.timelineSemaphore = VK_TRUE;
201 20 enabled.vulkan13.synchronization2 = VK_TRUE;
202 20 enabled.vulkan13.subgroupSizeControl = VK_TRUE;
203 20 enabled.vulkan13.computeFullSubgroups = VK_TRUE;
204
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 enabled.vulkan13.dynamicRendering = extensions.presentation ? VK_TRUE : VK_FALSE;
205
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 enabled.accelerationStructure.accelerationStructure = hardware ? VK_TRUE : VK_FALSE;
206 20 enabled.rayQuery.rayQuery = hardware ? VK_TRUE : VK_FALSE;
207
208
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 19 times.
 20 const VkBool32 bindless = textures ? VK_TRUE : VK_FALSE;
209 20 enabled.vulkan12.runtimeDescriptorArray = bindless;
210 20 enabled.vulkan12.descriptorBindingPartiallyBound = bindless;
211 20 enabled.vulkan12.descriptorBindingSampledImageUpdateAfterBind = bindless;
212 20 enabled.vulkan12.descriptorBindingUpdateUnusedWhilePending = bindless;
213 20 enabled.vulkan12.shaderSampledImageArrayNonUniformIndexing = bindless;
214
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 19 times.
 20 enabled.base.features.textureCompressionBC = textures && blockCompression ? VK_TRUE : VK_FALSE;
215
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 19 times.
 20 enabled.base.features.samplerAnisotropy = textures && anisotropy ? VK_TRUE : VK_FALSE;
216
217
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 const float priorities[2] = {queueCount > 1 ? 0.5f : 1.0f, 1.0f};
218 20 VkDeviceQueueCreateInfo queue{VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};
219 20 queue.queueFamilyIndex = queueFamily;
220 20 queue.queueCount = queueCount;
221 20 queue.pQueuePriorities = priorities;
222 20 VkDeviceCreateInfo info{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, &enabled.base};
223 20 info.queueCreateInfoCount = 1;
224 20 info.pQueueCreateInfos = &queue;
225
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 18 times.
 20 info.enabledExtensionCount = uint32_t(names.size());
226
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 18 times.
 20 info.ppEnabledExtensionNames = names.data();
227 20 VkDevice device = VK_NULL_HANDLE;
228
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 18 times.
 20 if (creator != nullptr) {
229 2 device = creator->createDevice(vkGetInstanceProcAddr, physicalDevice, info);
230
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 2 times.
 2 if (device == VK_NULL_HANDLE) {
231
0/2
✗ Branch 2 not taken.
✗ Branch 5 not taken.
 ✗ throw Error("the Vulkan creator made no device");
232 }
233 return device;
234 }
235 18 Check(vkCreateDevice(physicalDevice, &info, nullptr, &device), "vkCreateDevice");
236 18 return device;
237 }
238
239 20 VmaAllocator CreateAllocator(VkInstance instance, VkPhysicalDevice physicalDevice,
240 VkDevice device) {
241 20 VmaVulkanFunctions functions{};
242 20 functions.vkGetInstanceProcAddr = vkGetInstanceProcAddr;
243 20 functions.vkGetDeviceProcAddr = vkGetDeviceProcAddr;
244 20 VmaAllocatorCreateInfo info{};
245 20 info.flags = VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT;
246 20 info.vulkanApiVersion = VK_API_VERSION_1_3;
247 20 info.physicalDevice = physicalDevice;
248 20 info.device = device;
249 20 info.instance = instance;
250 20 info.pVulkanFunctions = &functions;
251 20 VmaAllocator allocator = VK_NULL_HANDLE;
252 20 Check(vmaCreateAllocator(&info, &allocator), "vmaCreateAllocator");
253 20 return allocator;
254 }
255
256 40 std::string PipelineCacheKey(const AdapterInfo& adapter) {
257 40 const std::string_view uuid(reinterpret_cast<const char*>(adapter.uuid.data()),
258 40 adapter.uuid.size());
259
1/1
✓ Branch 1 taken 40 times.
 80 return fmt::format("vulkan pipelines {} {} driver {}", adapter.name, Digest(uuid),
260
1/1
✓ Branch 2 taken 40 times.
 120 adapter.driverVersion.ToString());
261 }
262
263 }
264
265 20 VulkanDevice::VulkanDevice(const AdapterInfo& adapter, const DeviceDesc& desc, Traversal traversal,
266 20 OwnedInstance instance)
267
2/2
✓ Branch 2 taken 20 times.
✓ Branch 7 taken 20 times.
 20 : Device(adapter, desc, traversal), m_ownedInstance(std::move(instance)) {
268
1/1
✓ Branch 1 taken 20 times.
 20 GetApi();
269 20 try {
270
2/2
✓ Branch 1 taken 20 times.
✓ Branch 4 taken 20 times.
 20 m_labels = Extensions().Has(VK_EXT_DEBUG_UTILS_EXTENSION_NAME);
271
2/2
✓ Branch 0 taken 18 times.
✓ Branch 1 taken 2 times.
 20 m_instance = m_ownedInstance.Get() != VK_NULL_HANDLE ? m_ownedInstance.Get()
272
1/1
✓ Branch 1 taken 18 times.
 18 : vulkan::Instance(desc.validation);
273
1/1
✓ Branch 1 taken 20 times.
 20 m_physicalDevice = FindPhysicalDevice(m_instance, adapter);
274 40 const VkQueueFlags flags =
275
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 VK_QUEUE_COMPUTE_BIT | (desc.presentation ? VK_QUEUE_GRAPHICS_BIT : 0);
276
1/1
✓ Branch 1 taken 20 times.
 20 const std::optional<ComputeFamily> family = ComputeQueueFamily(m_physicalDevice, flags);
277
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 if (!family) {
278 ✗ throw Error("the Vulkan adapter {} has no {} queue", adapter.name,
279
0/3
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 4 not taken.
 ✗ desc.presentation ? "graphics and compute" : "compute");
280 }
281
282
1/1
✓ Branch 1 taken 20 times.
 20 m_queueFamily = family->index;
283 20 VkPhysicalDeviceProperties properties{};
284
1/1
✓ Branch 1 taken 20 times.
 20 vkGetPhysicalDeviceProperties(m_physicalDevice, &properties);
285 20 m_vendorId = properties.vendorID;
286 20 m_timestampPeriod = properties.limits.timestampPeriod;
287
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 const uint32_t bits = family->properties.timestampValidBits;
288
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 m_timestampMask = bits >= 64 ? ~uint64_t(0) : (uint64_t(1) << bits) - 1;
289
290 20 const bool hardware = traversal == Traversal::Hardware;
291 20 const DeviceExtensions extensions =
292
1/1
✓ Branch 1 taken 20 times.
 20 ChooseExtensions(m_physicalDevice, adapter, desc, hardware);
293 20 m_sharesSemaphores = extensions.sharesSemaphores;
294 20 m_sharesMemory = extensions.sharesMemory;
295 20 m_capturesStatistics = extensions.capturesStatistics;
296
297
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 const bool textures = GetSampling() == Sampling::Hardware;
298
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 20 const uint32_t queueCount = desc.presentation && family->properties.queueCount > 1 ? 2 : 1;
299 60 m_device = CreateLogicalDevice(
300 m_physicalDevice, m_queueFamily, queueCount, extensions, hardware, textures,
301
1/1
✓ Branch 1 taken 20 times.
 20 adapter.features.blockCompression, adapter.features.anisotropicFiltering,
302
2/2
✓ Branch 0 taken 2 times.
✓ Branch 1 taken 18 times.
 20 m_ownedInstance.Get() != VK_NULL_HANDLE ? desc.creator.get() : nullptr);
303
1/1
✓ Branch 1 taken 20 times.
 20 LoadDeviceFunctions(m_functions, m_device);
304
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 if (extensions.calibrated != nullptr) {
305 40 m_calibratedTimestamps = std::string_view(extensions.calibrated).starts_with("VK_KHR")
306
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 ? m_functions.vkGetCalibratedTimestampsKHR
307 : m_functions.vkGetCalibratedTimestampsEXT;
308 }
309
1/1
✓ Branch 1 taken 20 times.
 20 m_functions.vkGetDeviceQueue(m_device, m_queueFamily, 0, &m_queueHandle);
310
1/1
✓ Branch 1 taken 20 times.
 20 m_allocator = CreateAllocator(m_instance, m_physicalDevice, m_device);
311
312 20 const std::optional<std::string> saved =
313
4/4
✓ Branch 1 taken 20 times.
✓ Branch 4 taken 20 times.
✓ Branch 7 taken 20 times.
✓ Branch 10 taken 20 times.
 100 DiskCache(KernelCacheDirectory() / "vulkan").Load(PipelineCacheKey(Adapter()));
314 20 VkPipelineCacheCreateInfo pipelines{VK_STRUCTURE_TYPE_PIPELINE_CACHE_CREATE_INFO};
315
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 pipelines.initialDataSize = saved ? saved->size() : 0;
316
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 pipelines.pInitialData = saved ? saved->data() : nullptr;
317
2/2
✓ Branch 1 taken 20 times.
✓ Branch 4 taken 20 times.
 20 Check(m_functions.vkCreatePipelineCache(m_device, &pipelines, nullptr, &m_pipelineCache),
318 "vkCreatePipelineCache");
319
320
1/1
✓ Branch 1 taken 20 times.
 20 CreateDescriptors(textures);
321
322
1/1
✓ Branch 1 taken 20 times.
 20 Log(LogLevel::Debug, "created a Vulkan device on queue family {}{}{}", m_queueFamily,
323
2/3
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
✓ Branch 3 taken 20 times.
 40 extensions.names.empty() ? "" : ", with ",
324
3/3
✓ Branch 1 taken 20 times.
✓ Branch 4 taken 20 times.
✓ Branch 8 taken 20 times.
 40 Join({extensions.names.begin(), extensions.names.end()}));
325
1/1
✓ Branch 1 taken 20 times.
 20 m_queue = CreateQueue(*this, m_queueHandle, true);
326
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 if (queueCount > 1) {
327 ✗ VkQueue display = VK_NULL_HANDLE;
328
0/1
✗ Branch 1 not taken.
 ✗ m_functions.vkGetDeviceQueue(m_device, m_queueFamily, 1, &display);
329
0/1
✗ Branch 1 not taken.
 ✗ m_displayQueue = CreateQueue(*this, display, false);
330 }
331 20 } catch (...) {
332
0/1
✗ Branch 1 not taken.
 ✗ Destroy();
333 ✗ throw;
334 ✗ }
335 20 }
336
337 20 void VulkanDevice::CreateDescriptors(bool textures) {
338
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 1 time.
 20 const uint32_t textureSlots = textures ? Adapter().limits.maxTextures : 0;
339 19 if (textures) {
340 19 CreateSamplers();
341 }
342
343 20 VkDescriptorSetLayoutBinding bindings[3] = {};
344 20 bindings[0].binding = 0;
345 20 bindings[0].descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER;
346 20 bindings[0].descriptorCount = 1;
347 20 bindings[0].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
348 20 bindings[1].binding = kTextureBinding;
349 20 bindings[1].descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
350 20 bindings[1].descriptorCount = textureSlots;
351 20 bindings[1].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
352 20 bindings[2].binding = kSamplerBinding;
353 20 bindings[2].descriptorType = VK_DESCRIPTOR_TYPE_SAMPLER;
354 20 bindings[2].descriptorCount = uint32_t(m_samplers.size());
355 20 bindings[2].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT;
356
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 19 times.
 20 bindings[2].pImmutableSamplers = m_samplers.data();
357 20 const VkDescriptorBindingFlags flags[3] = {
358 0,
359 VK_DESCRIPTOR_BINDING_PARTIALLY_BOUND_BIT | VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT |
360 VK_DESCRIPTOR_BINDING_UPDATE_UNUSED_WHILE_PENDING_BIT,
361 0};
362 20 VkDescriptorSetLayoutBindingFlagsCreateInfo bindingFlags{
363 VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO};
364 20 bindingFlags.bindingCount = 3;
365 20 bindingFlags.pBindingFlags = flags;
366 20 VkDescriptorSetLayoutCreateInfo set{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO};
367
2/2
✓ Branch 0 taken 1 time.
✓ Branch 1 taken 19 times.
 20 set.bindingCount = textures ? 3 : 1;
368 20 set.pBindings = bindings;
369
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 1 time.
 20 if (textures) {
370 19 set.pNext = &bindingFlags;
371 19 set.flags = VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT;
372 }
373 20 Check(m_functions.vkCreateDescriptorSetLayout(m_device, &set, nullptr, &m_globalsLayout),
374 "vkCreateDescriptorSetLayout");
375
376 20 const VkPushConstantRange params{VK_SHADER_STAGE_COMPUTE_BIT, 0,
377 20 Adapter().limits.maxParamBytes};
378 20 VkPipelineLayoutCreateInfo layout{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO};
379 20 layout.setLayoutCount = 1;
380 20 layout.pSetLayouts = &m_globalsLayout;
381 20 layout.pushConstantRangeCount = 1;
382 20 layout.pPushConstantRanges = &params;
383 20 Check(m_functions.vkCreatePipelineLayout(m_device, &layout, nullptr, &m_pipelineLayout),
384 "vkCreatePipelineLayout");
385
386 20 const VkDescriptorPoolSize sizes[3] = {{VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER, 1},
387 {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, textureSlots},
388 20 {VK_DESCRIPTOR_TYPE_SAMPLER, kTextureSamplers}};
389 20 VkDescriptorPoolCreateInfo pool{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO};
390 20 pool.maxSets = 1;
391 20 pool.poolSizeCount = textures ? 3 : 1;
392 20 pool.pPoolSizes = sizes;
393
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 1 time.
 20 if (textures) {
394 19 pool.flags = VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT;
395 }
396 20 Check(m_functions.vkCreateDescriptorPool(m_device, &pool, nullptr, &m_descriptorPool),
397 "vkCreateDescriptorPool");
398
399 20 VkDescriptorSetAllocateInfo allocate{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO};
400 20 allocate.descriptorPool = m_descriptorPool;
401 20 allocate.descriptorSetCount = 1;
402 20 allocate.pSetLayouts = &m_globalsLayout;
403 20 Check(m_functions.vkAllocateDescriptorSets(m_device, &allocate, &m_globals),
404 "vkAllocateDescriptorSets");
405 20 }
406
407 121128 void VulkanDevice::Name(VkObjectType type, uint64_t object, const std::string& name) const {
408
2/4
✓ Branch 0 taken 121128 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 121128 times.
 121128 if (!m_labels || name.empty()) {
409 ✗ return;
410 }
411 121128 VkDebugUtilsObjectNameInfoEXT info{VK_STRUCTURE_TYPE_DEBUG_UTILS_OBJECT_NAME_INFO_EXT};
412 121128 info.objectType = type;
413 121128 info.objectHandle = object;
414 121128 info.pObjectName = name.c_str();
415 121128 Check(vkSetDebugUtilsObjectNameEXT(m_device, &info), "vkSetDebugUtilsObjectNameEXT");
416 }
417
418 11 bool VulkanDevice::ToNanoseconds(std::span<uint64_t> timestamps) const {
419 11 const auto calibrated = CalibratedTimestamps();
420
2/2
✓ Branch 0 taken 36 times.
✓ Branch 1 taken 11 times.
 47 for (uint64_t& time : timestamps) {
421 36 const auto ticks = int64_t(time & m_timestampMask);
422
1/2
✓ Branch 0 taken 36 times.
✗ Branch 1 not taken.
 72 time = calibrated
423
1/2
✓ Branch 0 taken 36 times.
✗ Branch 1 not taken.
 36 ? calibrated->second +
424 36 uint64_t(double(ticks - int64_t(calibrated->first & m_timestampMask)) *
425 36 m_timestampPeriod)
426 ✗ : uint64_t(double(ticks) * m_timestampPeriod);
427 }
428 11 return calibrated.has_value();
429 }
430
431 11 std::optional<std::pair<uint64_t, uint64_t>> VulkanDevice::CalibratedTimestamps() const {
432
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 11 times.
 11 if (m_calibratedTimestamps == nullptr) {
433 ✗ return std::nullopt;
434 }
435 11 const VkCalibratedTimestampInfoKHR domains[2] = {
436 {VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, nullptr, VK_TIME_DOMAIN_DEVICE_KHR},
437 {VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_KHR, nullptr, kHostTimeDomain}};
438 11 uint64_t times[2] = {};
439 11 uint64_t deviation = 0;
440 11 Check(m_calibratedTimestamps(m_device, 2, domains, times, &deviation),
441 "vkGetCalibratedTimestampsKHR");
442 11 return std::pair{times[0], HostClockNs(times[1])};
443 }
444
445 40 VulkanDevice::~VulkanDevice() {
446 20 Destroy();
447 40 }
448
449 20 void VulkanDevice::Destroy() {
450
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 if (m_displayQueue) {
451 ✗ CloseQueue(*m_displayQueue);
452
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ m_displayQueue.reset();
453 }
454
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 if (m_device != VK_NULL_HANDLE) {
455 20 m_functions.vkDestroyDescriptorPool(m_device, m_descriptorPool, nullptr);
456 20 m_functions.vkDestroyPipelineLayout(m_device, m_pipelineLayout, nullptr);
457 20 m_functions.vkDestroyDescriptorSetLayout(m_device, m_globalsLayout, nullptr);
458 20 m_descriptorPool = VK_NULL_HANDLE;
459 20 m_pipelineLayout = VK_NULL_HANDLE;
460 20 m_globalsLayout = VK_NULL_HANDLE;
461
2/2
✓ Branch 0 taken 320 times.
✓ Branch 1 taken 20 times.
 340 for (VkSampler& sampler : m_samplers) {
462 320 m_functions.vkDestroySampler(m_device, sampler, nullptr);
463 320 sampler = VK_NULL_HANDLE;
464 }
465 }
466
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 if (m_pipelineCache != VK_NULL_HANDLE) {
467 60 const std::vector<char> data = Enumerate<char, size_t>([this](size_t* bytes, char* items) {
468 40 m_functions.vkGetPipelineCacheData(m_device, m_pipelineCache, bytes, items);
469 20 });
470
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 if (!data.empty()) {
471
3/3
✓ Branch 1 taken 20 times.
✓ Branch 4 taken 20 times.
✓ Branch 7 taken 20 times.
 100 DiskCache(KernelCacheDirectory() / "vulkan")
472
2/2
✓ Branch 1 taken 20 times.
✓ Branch 4 taken 20 times.
 40 .Store(PipelineCacheKey(Adapter()), {data.data(), data.size()});
473 }
474
1/1
✓ Branch 1 taken 20 times.
 20 m_functions.vkDestroyPipelineCache(m_device, m_pipelineCache, nullptr);
475 20 m_pipelineCache = VK_NULL_HANDLE;
476 20 }
477 20 ReleaseSpares();
478
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 m_memoryInterop.reset();
479
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 9 times.
 20 if (m_interopPool != VK_NULL_HANDLE) {
480 11 vmaDestroyPool(m_allocator, m_interopPool);
481 11 m_interopPool = VK_NULL_HANDLE;
482 }
483
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 if (m_allocator != VK_NULL_HANDLE) {
484 20 vmaDestroyAllocator(m_allocator);
485 20 m_allocator = VK_NULL_HANDLE;
486 }
487
1/2
✓ Branch 0 taken 20 times.
✗ Branch 1 not taken.
 20 if (m_device != VK_NULL_HANDLE) {
488 20 m_functions.vkDestroyDevice(m_device, nullptr);
489 20 m_device = VK_NULL_HANDLE;
490 }
491 20 m_ownedInstance = {};
492 20 }
493
494 119994 VkBufferUsageFlags VulkanDevice::UsageFlags(BufferUsage usage) const {
495 // NOLINTNEXTLINE(bugprone-signed-bitwise)
496 119994 VkBufferUsageFlags flags = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT |
497 VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT |
498 VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT |
499 VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT;
500
2/2
✓ Branch 0 taken 842 times.
✓ Branch 1 taken 119152 times.
 119994 if (Has(usage, BufferUsage::Indirect)) {
501 842 flags |= VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT;
502 }
503
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 119994 times.
 119994 if (GetTraversal() == Traversal::Hardware) {
504
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (Has(usage, BufferUsage::AccelBuildInput)) {
505 ✗ flags |= VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_BUILD_INPUT_READ_ONLY_BIT_KHR;
506 }
507
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (Has(usage, BufferUsage::AccelStorage)) {
508 ✗ flags |= VK_BUFFER_USAGE_ACCELERATION_STRUCTURE_STORAGE_BIT_KHR;
509 }
510 }
511 119994 return flags;
512 }
513
514 namespace {
515
516 ✗ VkAccelerationStructureTypeKHR VulkanType(AccelLevel level) {
517 ✗ return level == AccelLevel::Top ? VK_ACCELERATION_STRUCTURE_TYPE_TOP_LEVEL_KHR
518 ✗ : VK_ACCELERATION_STRUCTURE_TYPE_BOTTOM_LEVEL_KHR;
519 }
520
521 }
522
523 ✗ void VulkanBuild::Describe(const AccelInput& input) {
524
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ info.type = VulkanType(GetLevel(input));
525 ✗ info.flags = VK_BUILD_ACCELERATION_STRUCTURE_PREFER_FAST_TRACE_BIT_KHR;
526
0/4
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ if (Refittable(input)) {
527 ✗ info.flags |= VK_BUILD_ACCELERATION_STRUCTURE_ALLOW_UPDATE_BIT_KHR;
528 }
529 ✗ info.mode = VK_BUILD_ACCELERATION_STRUCTURE_MODE_BUILD_KHR;
530 ✗ info.geometryCount = 1;
531 ✗ info.pGeometries = &geometry;
532 ✗ geometry.flags = VK_GEOMETRY_OPAQUE_BIT_KHR;
533 ✗ std::visit(Overloaded{
534 ✗ [&](const AccelTriangles& t) {
535 ✗ geometry.geometryType = VK_GEOMETRY_TYPE_TRIANGLES_KHR;
536 ✗ VkAccelerationStructureGeometryTrianglesDataKHR& triangles =
537 ✗ geometry.geometry.triangles;
538 ✗ triangles.sType =
539 VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_TRIANGLES_DATA_KHR;
540 ✗ triangles.vertexFormat = VK_FORMAT_R32G32B32_SFLOAT;
541
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ triangles.vertexData.deviceAddress = t.vertices.Address();
542 ✗ triangles.vertexStride = t.stride;
543
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ triangles.maxVertex = t.vertexCount > 0 ? t.vertexCount - 1 : 0;
544 ✗ triangles.indexType =
545
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ t.indices ? VK_INDEX_TYPE_UINT32 : VK_INDEX_TYPE_NONE_KHR;
546
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ triangles.indexData.deviceAddress = t.indices ? t.indices->Address() : 0;
547 ✗ primitives = t.count;
548 ✗ },
549 ✗ [&](const AccelBoxes& b) {
550 ✗ geometry.geometryType = VK_GEOMETRY_TYPE_AABBS_KHR;
551 ✗ geometry.geometry.aabbs.sType =
552 VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_AABBS_DATA_KHR;
553 ✗ geometry.geometry.aabbs.data.deviceAddress = b.boxes.Address();
554 ✗ geometry.geometry.aabbs.stride = 6 * sizeof(float);
555 ✗ primitives = b.count;
556 ✗ },
557 ✗ [&](const AccelInstances& i) {
558 ✗ geometry.geometryType = VK_GEOMETRY_TYPE_INSTANCES_KHR;
559 ✗ geometry.geometry.instances.sType =
560 VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_INSTANCES_DATA_KHR;
561 ✗ geometry.geometry.instances.arrayOfPointers = VK_FALSE;
562 ✗ geometry.geometry.instances.data.deviceAddress = i.instances.Address();
563 ✗ primitives = i.count;
564 ✗ },
565 },
566 input);
567 ✗ }
568
569 ✗ AccelSizes VulkanDevice::HardwareBuildSizes(const AccelInput& input) const {
570 ✗ VulkanBuild build;
571 ✗ build.Describe(input);
572 ✗ VkAccelerationStructureBuildSizesInfoKHR sizes{
573 VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_SIZES_INFO_KHR};
574 ✗ m_functions.vkGetAccelerationStructureBuildSizesKHR(
575 ✗ m_device, VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR, &build.info, &build.primitives,
576 &sizes);
577 ✗ return {.storage = sizes.accelerationStructureSize,
578 ✗ .scratch = sizes.buildScratchSize,
579
0/4
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ .updateScratch = Refittable(input) ? sizes.updateScratchSize : 0};
580 }
581
582 ✗ Ref<Accel> VulkanDevice::NewAccel(Ref<Buffer> storage, AccelLevel level, AccelSizes sizes,
583 std::string label) {
584
0/3
✗ Branch 3 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 ✗ return Ref<Accel>(new VulkanAccel(*this, std::move(storage), level, sizes, std::move(label)));
585 }
586
587 ✗ VulkanAccel::VulkanAccel(VulkanDevice& owner, Ref<Buffer> storage, AccelLevel level,
588 ✗ AccelSizes sizes, std::string label)
589
0/1
✗ Branch 2 not taken.
 ✗ : Accel(owner, std::move(storage), level, sizes, std::move(label)), m_owner(owner) {
590 ✗ VkAccelerationStructureCreateInfoKHR info{
591 VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_CREATE_INFO_KHR};
592
0/1
✗ Branch 1 not taken.
 ✗ info.buffer = static_cast<VulkanBuffer&>(Storage()).Handle();
593
0/1
✗ Branch 1 not taken.
 ✗ info.size = Storage().Bytes();
594 ✗ info.type = VulkanType(level);
595
0/1
✗ Branch 1 not taken.
 ✗ const VolkDeviceTable& vk = owner.Functions();
596
0/2
✗ Branch 1 not taken.
✗ Branch 4 not taken.
 ✗ Check(vk.vkCreateAccelerationStructureKHR(owner.Handle(), &info, nullptr, &m_handle),
597 "vkCreateAccelerationStructureKHR");
598 ✗ VkAccelerationStructureDeviceAddressInfoKHR address{
599 VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_DEVICE_ADDRESS_INFO_KHR};
600 ✗ address.accelerationStructure = m_handle;
601
0/1
✗ Branch 1 not taken.
 ✗ m_address = vk.vkGetAccelerationStructureDeviceAddressKHR(owner.Handle(), &address);
602
0/1
✗ Branch 1 not taken.
 ✗ owner.Name(VK_OBJECT_TYPE_ACCELERATION_STRUCTURE_KHR, uint64_t(m_handle), Label());
603 ✗ }
604
605 ✗ VulkanAccel::~VulkanAccel() {
606 ✗ m_owner.Functions().vkDestroyAccelerationStructureKHR(m_owner.Handle(), m_handle, nullptr);
607 ✗ }
608
609 119983 uint64_t VulkanDevice::BufferAddress(VkBuffer buffer) const {
610 119983 VkBufferDeviceAddressInfo info{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO};
611 119983 info.buffer = buffer;
612 119983 return m_functions.vkGetBufferDeviceAddress(m_device, &info);
613 }
614
615 889 VmaPool VulkanDevice::InteropPool() {
616 889 const std::lock_guard lock(m_interopMutex);
617
2/2
✓ Branch 0 taken 11 times.
✓ Branch 1 taken 878 times.
 889 if (m_interopPool != VK_NULL_HANDLE) {
618 return m_interopPool;
619 }
620
621 11 VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
622 11 external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
623 11 VkBufferCreateInfo sample{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external};
624 11 sample.size = 65536;
625 11 sample.usage = UsageFlags(BufferUsage::None);
626 11 VmaAllocationCreateInfo wanted{};
627 11 wanted.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
628 11 uint32_t type = 0;
629
2/2
✓ Branch 1 taken 11 times.
✓ Branch 4 taken 11 times.
 11 Check(vmaFindMemoryTypeIndexForBufferInfo(m_allocator, &sample, &wanted, &type),
630 "vmaFindMemoryTypeIndexForBufferInfo");
631
632 11 m_exportInfo.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
633 11 VmaPoolCreateInfo pool{};
634 11 pool.memoryTypeIndex = type;
635 11 pool.pMemoryAllocateNext = &m_exportInfo;
636
2/2
✓ Branch 1 taken 11 times.
✓ Branch 4 taken 11 times.
 11 Check(vmaCreatePool(m_allocator, &pool, &m_interopPool), "vmaCreatePool");
637 11 return m_interopPool;
638 889 }
639
640 2971 MemoryInterop* VulkanDevice::GetMemoryInterop() {
641 2971 const std::lock_guard lock(m_interopMutex);
642
2/2
✓ Branch 0 taken 13 times.
✓ Branch 1 taken 2958 times.
 2971 if (!m_interopTried) {
643 13 m_interopTried = true;
644
1/2
✓ Branch 0 taken 13 times.
✗ Branch 1 not taken.
 13 if (m_sharesMemory) {
645 13 try {
646
0/1
✗ Branch 1 not taken.
 13 m_memoryInterop = CreateMemoryInterop(*this);
647
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 13 times.
 13 } catch (const UnsupportedError& error) {
648
1/1
✓ Branch 2 taken 13 times.
 13 Log(LogLevel::Debug, "{} shares no memory: {}", Adapter().alias, error.what());
649 13 }
650 }
651 }
652 2971 return m_memoryInterop.get();
653 2971 }
654
655 1193 std::optional<DLDevice> VulkanDevice::SharedDevice() {
656 1193 const MemoryInterop* interop = GetMemoryInterop();
657
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1193 times.
 1193 return interop != nullptr ? std::optional(interop->Device()) : std::nullopt;
658 }
659
660 ✗ Stream VulkanDevice::CopyStream() {
661 ✗ MemoryInterop* interop = GetMemoryInterop();
662
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ return interop != nullptr ? Stream::Compute(interop->Stream()) : Stream::None();
663 }
664
665 ✗ void VulkanDevice::ExternalCopy(BufferRange dst, uint64_t source, Stream producer) {
666 ✗ MemoryInterop& interop = *GetMemoryInterop();
667 ✗ const ExternalAddress external = dst.GetBuffer().External(interop.Device().device_type);
668
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ interop.Copy(external.address + dst.Offset(), source, dst.Bytes(),
669
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ producer.IsCompute() ? producer.Handle() : nullptr);
670 ✗ }
671
672 239966 VulkanBuffer::~VulkanBuffer() {
673 119983 const bool reusable =
674
3/4
✓ Branch 0 taken 889 times.
✓ Branch 1 taken 119094 times.
✗ Branch 3 not taken.
✓ Branch 4 taken 889 times.
 119983 Has(Usage(), BufferUsage::Interop) && m_owner.GetMemoryInterop() != nullptr;
675
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (reusable && m_owner.KeepSpare({m_buffer, m_allocation, m_external, m_externalAddress,
676 ✗ Bytes(), Usage()})) {
677 ✗ return;
678 }
679
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 119983 times.
 119983 if (m_external != nullptr) {
680 ✗ ImportedMemory imported{m_external, m_externalAddress};
681 ✗ m_owner.GetMemoryInterop()->Release(imported);
682 }
683 119983 vmaDestroyBuffer(m_owner.Allocator(), m_buffer, m_allocation);
684 239966 }
685
686 namespace {
687
688 ✗ ExportedMemory ExportMemory(const VulkanDevice& owner, VkBuffer buffer, VmaAllocation allocation,
689 uint64_t bytes) {
690 ✗ VmaAllocationInfo info{};
691 ✗ vmaGetAllocationInfo(owner.Allocator(), allocation, &info);
692 ✗ VkMemoryGetFdInfoKHR get{VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR};
693 ✗ get.memory = info.deviceMemory;
694 ✗ get.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
695 ✗ ExportedMemory exported;
696 ✗ Check(owner.Functions().vkGetMemoryFdKHR(owner.Handle(), &get, &exported.fd),
697 "vkGetMemoryFdKHR");
698
699 ✗ VkMemoryRequirements requirements{};
700 ✗ owner.Functions().vkGetBufferMemoryRequirements(owner.Handle(), buffer, &requirements);
701
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ exported.memoryBytes = std::max<uint64_t>(info.size, requirements.size);
702 ✗ exported.offset = info.offset;
703 ✗ exported.bytes = bytes;
704 ✗ return exported;
705 }
706
707 }
708
709 ✗ ExternalAddress VulkanBuffer::External(DLDeviceType api) {
710 ✗ MemoryInterop* interop = m_owner.GetMemoryInterop();
711
0/6
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
 ✗ if (interop == nullptr || interop->Device().device_type != api ||
712
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ !Has(Usage(), BufferUsage::Interop)) {
713 ✗ throw UnsupportedError("Buffer \"{}\" on {} isn't shared with device type {}", Label(),
714
0/1
✗ Branch 2 not taken.
 ✗ m_owner.Adapter().alias, int(api));
715 }
716
717 ✗ const std::lock_guard lock(m_externalMutex);
718
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (m_external == nullptr) {
719 ✗ const ImportedMemory imported =
720
0/2
✗ Branch 1 not taken.
✗ Branch 4 not taken.
 ✗ interop->Import(ExportMemory(m_owner, m_buffer, m_allocation, Bytes()));
721 ✗ m_external = imported.external;
722 ✗ m_externalAddress = imported.address;
723 }
724
0/1
✗ Branch 1 not taken.
 ✗ return {interop->Device(), m_externalAddress};
725 ✗ }
726
727 namespace {
728
729 119983 VmaAllocationCreateInfo MemoryAllocation(Memory memory) {
730 119983 constexpr VkMemoryPropertyFlags kCoherent =
731 VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
732 119983 VmaAllocationCreateInfo allocation{};
733
3/5
✓ Branch 0 taken 119808 times.
✓ Branch 1 taken 91 times.
✓ Branch 2 taken 84 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 119983 switch (memory) {
734 119808 case Memory::Device:
735 119808 allocation.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
736 119808 break;
737 91 case Memory::Upload:
738 91 allocation.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST;
739 91 allocation.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT |
740 VMA_ALLOCATION_CREATE_MAPPED_BIT;
741 91 allocation.requiredFlags = kCoherent;
742 91 break;
743 84 case Memory::Readback:
744 84 allocation.usage = VMA_MEMORY_USAGE_AUTO_PREFER_HOST;
745 84 allocation.flags =
746 VMA_ALLOCATION_CREATE_HOST_ACCESS_RANDOM_BIT | VMA_ALLOCATION_CREATE_MAPPED_BIT;
747 84 allocation.requiredFlags = kCoherent;
748 84 allocation.preferredFlags = VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
749 84 break;
750 ✗ case Memory::Shared:
751 ✗ allocation.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
752 ✗ allocation.flags = VMA_ALLOCATION_CREATE_HOST_ACCESS_SEQUENTIAL_WRITE_BIT |
753 VMA_ALLOCATION_CREATE_MAPPED_BIT;
754 ✗ allocation.requiredFlags = kCoherent | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
755 ✗ break;
756 case Memory::Virtual:
757 break;
758 }
759 119983 return allocation;
760 }
761
762 }
763
764 ✗ bool VulkanDevice::KeepSpare(const SpareBuffer& spare) {
765 ✗ constexpr size_t kMostSpares = 32;
766 ✗ constexpr uint64_t kMostSpareBytes = 256ull << 20;
767 ✗ const std::lock_guard lock(m_spareMutex);
768
0/4
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ if (m_sparesClosed || m_spares.size() >= kMostSpares ||
769
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ m_spareBytes + spare.bytes > kMostSpareBytes) {
770 return false;
771 }
772
0/1
✗ Branch 1 not taken.
 ✗ m_spares.push_back(spare);
773 ✗ m_spareBytes += spare.bytes;
774 ✗ return true;
775 ✗ }
776
777 889 std::optional<VulkanDevice::SpareBuffer> VulkanDevice::TakeSpare(uint64_t bytes,
778 BufferUsage usage) {
779 889 const std::lock_guard lock(m_spareMutex);
780 889 const auto found = std::ranges::find_if(m_spares, [&](const SpareBuffer& spare) {
781
0/4
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ return spare.bytes == bytes && spare.usage == usage;
782 });
783
1/2
✓ Branch 0 taken 889 times.
✗ Branch 1 not taken.
 889 if (found == m_spares.end()) {
784 889 return std::nullopt;
785 }
786 ✗ const SpareBuffer spare = *found;
787 ✗ m_spares.erase(found);
788 ✗ m_spareBytes -= spare.bytes;
789 ✗ return spare;
790 889 }
791
792 20 void VulkanDevice::ReleaseSpares() {
793 20 std::vector<SpareBuffer> spares;
794 20 {
795
1/1
✓ Branch 1 taken 20 times.
 20 const std::lock_guard lock(m_spareMutex);
796 20 m_sparesClosed = true;
797 20 spares.swap(m_spares);
798 20 m_spareBytes = 0;
799 20 }
800
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 20 times.
 20 for (const SpareBuffer& spare : spares) {
801
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (spare.external != nullptr) {
802 ✗ ImportedMemory imported{spare.external, spare.externalAddress};
803 ✗ m_memoryInterop->Release(imported);
804 }
805
0/1
✗ Branch 1 not taken.
 ✗ vmaDestroyBuffer(m_allocator, spare.buffer, spare.allocation);
806 }
807 20 }
808
809 10 std::vector<MemoryHeap> VulkanDevice::Heaps() const {
810 10 const VkPhysicalDeviceMemoryProperties* properties = nullptr;
811 10 vmaGetMemoryProperties(m_allocator, &properties);
812 10 std::vector<VmaBudget> budgets(properties->memoryHeapCount);
813
1/1
✓ Branch 1 taken 10 times.
 10 vmaGetHeapBudgets(m_allocator, budgets.data());
814 10 std::vector<MemoryHeap> heaps;
815
2/2
✓ Branch 0 taken 10 times.
✓ Branch 1 taken 10 times.
 20 for (uint32_t i = 0; i < properties->memoryHeapCount; ++i) {
816 10 const VkMemoryHeap& heap = properties->memoryHeaps[i];
817
1/1
✓ Branch 1 taken 10 times.
 10 heaps.push_back({.usage = budgets[i].usage,
818 10 .budget = budgets[i].budget,
819 10 .size = heap.size,
820
1/1
✓ Branch 1 taken 10 times.
 10 .deviceLocal = (heap.flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0});
821 }
822 10 return heaps;
823 10 }
824
825 119983 Ref<Buffer> VulkanDevice::NewBuffer(const BufferDesc& desc) {
826 119983 VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO};
827 119983 info.size = desc.bytes;
828 119983 info.usage = UsageFlags(desc.usage);
829 119983 info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
830 119983 VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
831 119983 const bool interop =
832
4/6
✓ Branch 0 taken 889 times.
✓ Branch 1 taken 119094 times.
✓ Branch 3 taken 889 times.
✗ Branch 4 not taken.
✓ Branch 5 taken 889 times.
✗ Branch 6 not taken.
 119983 Has(desc.usage, BufferUsage::Interop) && (GetMemoryInterop() != nullptr || m_sharesMemory);
833 if (interop) {
834
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 889 times.
 889 if (desc.memory != Memory::Device) {
835 ✗ throw UnsupportedError("Buffer \"{}\" asks for interop, which only device memory has",
836
0/1
✗ Branch 2 not taken.
 ✗ desc.label);
837 }
838 889 external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
839 889 info.pNext = &external;
840 }
841
842 889 if (interop) {
843
1/2
✗ Branch 1 not taken.
✓ Branch 2 taken 889 times.
 889 if (const std::optional<SpareBuffer> spare = TakeSpare(desc.bytes, desc.usage)) {
844 ✗ vmaSetAllocationName(m_allocator, spare->allocation, desc.label.c_str());
845 ✗ Name(VK_OBJECT_TYPE_BUFFER, uint64_t(spare->buffer), desc.label);
846
0/1
✗ Branch 1 not taken.
 ✗ auto* kept = new VulkanBuffer(*this, desc, spare->buffer, spare->allocation, nullptr,
847
0/1
✗ Branch 2 not taken.
 ✗ spare->external, spare->externalAddress);
848 ✗ kept->exportable = true;
849 ✗ return Ref<Buffer>(kept);
850 }
851 }
852
853 119983 VmaAllocationCreateInfo allocation = MemoryAllocation(desc.memory);
854
2/2
✓ Branch 0 taken 889 times.
✓ Branch 1 taken 119094 times.
 119983 if (interop) {
855 889 allocation.pool = InteropPool();
856 889 allocation.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
857 }
858
859 119983 VkBuffer buffer = VK_NULL_HANDLE;
860 119983 VmaAllocation allocated = VK_NULL_HANDLE;
861 119983 VmaAllocationInfo allocatedInfo{};
862 119983 Check(vmaCreateBufferWithAlignment(m_allocator, &info, &allocation, kBufferAlignment, &buffer,
863 &allocated, &allocatedInfo),
864 "vmaCreateBufferWithAlignment");
865 119983 vmaSetAllocationName(m_allocator, allocated, desc.label.c_str());
866 119983 Name(VK_OBJECT_TYPE_BUFFER, uint64_t(buffer), desc.label);
867
1/1
✓ Branch 2 taken 119983 times.
 119983 auto* created = new VulkanBuffer(*this, desc, buffer, allocated, allocatedInfo.pMappedData);
868 119983 created->exportable = interop;
869 119983 return Ref<Buffer>(created);
870 }
871
872 ✗ Ref<Buffer> VulkanDevice::Import(Buffer& foreign) {
873 ✗ auto* buffer = dynamic_cast<VulkanBuffer*>(&foreign);
874
0/8
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
 ✗ if (buffer == nullptr || &buffer->Owner() == this || !buffer->exportable || !m_sharesMemory ||
875
0/6
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
 ✗ !buffer->Owner().SharesExportedMemory() ||
876
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ buffer->Owner().Adapter().uuid != Adapter().uuid) {
877 ✗ return {};
878 }
879 ✗ return buffer->SharedWith(*this);
880 }
881
882 ✗ Ref<Buffer> VulkanBuffer::SharedWith(VulkanDevice& reader) {
883 ✗ const std::lock_guard lock(m_externalMutex);
884
0/4
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 ✗ for (const auto& [device, view] : m_shared) {
885
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (device == &reader) {
886 ✗ return view;
887 }
888 }
889 ✗ Ref<Buffer> view = reader.ImportMemory(m_owner, m_buffer, m_allocation,
890
0/1
✗ Branch 1 not taken.
 ✗ {.bytes = Bytes(), .label = Label() + " (shared)"});
891
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (view) {
892
0/3
✗ Branch 1 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 ✗ m_shared.emplace_back(&reader, view);
893 }
894
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ return view;
895 ✗ }
896
897 ✗ ImportedVulkanBuffer::~ImportedVulkanBuffer() {
898 ✗ m_owner.Functions().vkFreeMemory(m_owner.Handle(), m_memory, nullptr);
899 ✗ }
900
901 ✗ Ref<Buffer> VulkanDevice::ImportMemory(const VulkanDevice& owner, VkBuffer buffer,
902 VmaAllocation allocation, const BufferDesc& desc) {
903 #ifdef _WIN32
904 (void)owner;
905 (void)buffer;
906 (void)allocation;
907 (void)desc;
908 return {};
909 #else
910 ✗ const ExportedMemory exported = ExportMemory(owner, buffer, allocation, desc.bytes);
911 ✗ VmaAllocationInfo2 info{};
912 ✗ vmaGetAllocationInfo2(owner.Allocator(), allocation, &info);
913
914 ✗ VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO};
915 ✗ external.handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
916 ✗ VkBufferCreateInfo create{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external};
917 ✗ create.size = desc.bytes;
918 ✗ create.usage = UsageFlags(desc.usage);
919 ✗ create.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
920 ✗ VkBuffer view = VK_NULL_HANDLE;
921
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 ✗ if (m_functions.vkCreateBuffer(m_device, &create, nullptr, &view) != VK_SUCCESS) {
922 ✗ close(exported.fd);
923 ✗ return {};
924 }
925 ✗ VkMemoryDedicatedAllocateInfo dedicated{VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO};
926 ✗ dedicated.buffer = view;
927 ✗ VkImportMemoryFdInfoKHR import{VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR,
928
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ info.dedicatedMemory ? &dedicated : nullptr};
929 ✗ import.handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT;
930 ✗ import.fd = exported.fd;
931 ✗ VkMemoryAllocateFlagsInfo flags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, &import};
932 ✗ flags.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT;
933 ✗ VkMemoryAllocateInfo allocate{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &flags};
934 ✗ allocate.allocationSize = info.blockSize;
935 ✗ allocate.memoryTypeIndex = info.allocationInfo.memoryType;
936 ✗ VkDeviceMemory memory = VK_NULL_HANDLE;
937
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 ✗ if (m_functions.vkAllocateMemory(m_device, &allocate, nullptr, &memory) != VK_SUCCESS) {
938 ✗ close(exported.fd);
939 ✗ m_functions.vkDestroyBuffer(m_device, view, nullptr);
940 ✗ return {};
941 }
942
0/2
✗ Branch 1 not taken.
✗ Branch 2 not taken.
 ✗ if (m_functions.vkBindBufferMemory(m_device, view, memory, info.allocationInfo.offset) !=
943 VK_SUCCESS) {
944 ✗ m_functions.vkFreeMemory(m_device, memory, nullptr);
945 ✗ m_functions.vkDestroyBuffer(m_device, view, nullptr);
946 ✗ return {};
947 }
948 ✗ Name(VK_OBJECT_TYPE_BUFFER, uint64_t(view), desc.label);
949
0/1
✗ Branch 2 not taken.
 ✗ return Ref<Buffer>(new ImportedVulkanBuffer(*this, desc, view, memory));
950 #endif
951 }
952
953 namespace {
954
955 ✗ std::string Value(const VkPipelineExecutableStatisticKHR& statistic) {
956
0/5
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
 ✗ switch (statistic.format) {
957 ✗ case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_BOOL32_KHR:
958
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ return statistic.value.b32 == VK_TRUE ? "true" : "false";
959 ✗ case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_INT64_KHR:
960 ✗ return std::to_string(statistic.value.i64);
961 ✗ case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR:
962 ✗ return std::to_string(statistic.value.u64);
963 ✗ case VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_FLOAT64_KHR:
964 ✗ return fmt::format("{}", statistic.value.f64);
965 ✗ default:
966 ✗ return "";
967 }
968 }
969
970 struct KnownStatistic {
971 const char* name;
972 std::optional<uint32_t> KernelStats::* field;
973 };
974
975 constexpr KnownStatistic kKnownStatistics[] = {
976 {"Register Count", &KernelStats::registers},
977 {"VGPRs", &KernelStats::registers},
978 {"Scratch Memory Size", &KernelStats::spillBytes},
979 {"Shared Memory Size", &KernelStats::sharedBytes},
980 {"Workgroup Memory Size", &KernelStats::sharedBytes},
981 };
982
983 1117 VkShaderModule CreateShaderModule(const VolkDeviceTable& functions, VkDevice device,
984 const KernelCode& code) {
985 1117 std::vector<uint32_t> words(code.code.size() / 4);
986
1/1
✓ Branch 1 taken 1117 times.
 1117 std::memcpy(words.data(), code.code.data(), words.size() * 4);
987 1117 VkShaderModuleCreateInfo shader{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO};
988 1117 shader.codeSize = words.size() * 4;
989 1117 shader.pCode = words.data();
990 1117 VkShaderModule module = VK_NULL_HANDLE;
991
2/2
✓ Branch 1 taken 1117 times.
✓ Branch 4 taken 1117 times.
 1117 Check(functions.vkCreateShaderModule(device, &shader, nullptr, &module),
992 "vkCreateShaderModule");
993 1117 return module;
994 1117 }
995
996 ✗ KernelStats ReadStatistics(const VolkDeviceTable& functions, VkDevice device, VkPipeline pipeline) {
997 ✗ const VkPipelineExecutableInfoKHR executable{VK_STRUCTURE_TYPE_PIPELINE_EXECUTABLE_INFO_KHR,
998 ✗ nullptr, pipeline, 0};
999 ✗ uint32_t count = 0;
1000 ✗ Check(functions.vkGetPipelineExecutableStatisticsKHR(device, &executable, &count, nullptr),
1001 "vkGetPipelineExecutableStatisticsKHR");
1002 ✗ std::vector<VkPipelineExecutableStatisticKHR> statistics(
1003
0/1
✗ Branch 2 not taken.
 ✗ count, {VK_STRUCTURE_TYPE_PIPELINE_EXECUTABLE_STATISTIC_KHR});
1004
0/2
✗ Branch 1 not taken.
✗ Branch 4 not taken.
 ✗ Check(functions.vkGetPipelineExecutableStatisticsKHR(device, &executable, &count,
1005 statistics.data()),
1006 "vkGetPipelineExecutableStatisticsKHR");
1007
1008 ✗ KernelStats stats{.cached = true};
1009
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ for (const VkPipelineExecutableStatisticKHR& statistic : statistics) {
1010
0/1
✗ Branch 1 not taken.
 ✗ stats.driver.push_back({statistic.name, Value(statistic)});
1011
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ for (const KnownStatistic& known : kKnownStatistics) {
1012
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ if (std::string_view(statistic.name) == known.name &&
1013
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ statistic.format == VK_PIPELINE_EXECUTABLE_STATISTIC_FORMAT_UINT64_KHR) {
1014 ✗ stats.*known.field = uint32_t(statistic.value.u64);
1015 }
1016 }
1017 }
1018 ✗ return stats;
1019 ✗ }
1020
1021 }
1022
1023 1117 KernelStats VulkanKernel::Load(const KernelCode& code, const DiskCache&, const DeviceGlobals&) {
1024 1117 const VolkDeviceTable& functions = m_owner.Functions();
1025 1117 const VkDevice device = m_owner.Handle();
1026 1117 m_owner.Globals();
1027 1117 const VkShaderModule module = CreateShaderModule(functions, device, code);
1028
1029 1117 VkComputePipelineCreateInfo pipeline{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO};
1030 1117 pipeline.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO};
1031 1117 pipeline.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT;
1032 1117 pipeline.stage.module = module;
1033
2/2
✓ Branch 0 taken 1102 times.
✓ Branch 1 taken 15 times.
 1117 pipeline.stage.pName = Name().c_str();
1034
2/2
✓ Branch 0 taken 1102 times.
✓ Branch 1 taken 15 times.
 1117 pipeline.layout = m_owner.PipelineLayout();
1035
2/2
✓ Branch 0 taken 1102 times.
✓ Branch 1 taken 15 times.
 1117 if (code.shape.groupSize.x % m_owner.Adapter().features.waveSizeMax == 0) {
1036 1102 pipeline.stage.flags |= VK_PIPELINE_SHADER_STAGE_CREATE_REQUIRE_FULL_SUBGROUPS_BIT;
1037 }
1038
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 1117 times.
 1117 if (m_owner.CapturesStatistics()) {
1039 ✗ pipeline.flags |= VK_PIPELINE_CREATE_CAPTURE_STATISTICS_BIT_KHR;
1040 }
1041 1117 const VkResult created = functions.vkCreateComputePipelines(device, m_owner.PipelineCache(), 1,
1042 &pipeline, nullptr, &m_pipeline);
1043 1117 functions.vkDestroyShaderModule(device, module, nullptr);
1044 1117 Check(created, "vkCreateComputePipelines");
1045
1/1
✓ Branch 1 taken 1117 times.
 1117 m_owner.Name(VK_OBJECT_TYPE_PIPELINE, uint64_t(m_pipeline),
1046 1117 fmt::format("{} of program \"{}\"", Name(), GetProgram().Label()));
1047
1048
1/2
✓ Branch 0 taken 1117 times.
✗ Branch 1 not taken.
 1117 if (!m_owner.CapturesStatistics()) {
1049 1117 return {.cached = true};
1050 }
1051 ✗ return ReadStatistics(functions, device, m_pipeline);
1052 }
1053
1054 2238 VulkanKernel::~VulkanKernel() {
1055 1119 m_owner.Functions().vkDestroyPipeline(m_owner.Handle(), m_pipeline, nullptr);
1056 2238 }
1057
1058 233727 VkDescriptorSet VulkanDevice::Globals() {
1059 233727 std::call_once(m_globalsWritten, [this] {
1060 17 VkDescriptorBufferInfo buffer{};
1061 17 buffer.buffer = static_cast<const VulkanBuffer&>(GlobalsBuffer()).Handle();
1062 17 buffer.range = sizeof(DeviceGlobals);
1063 17 VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
1064 17 write.dstSet = m_globals;
1065 17 write.descriptorCount = 1;
1066 17 write.descriptorType = VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER;
1067 17 write.pBufferInfo = &buffer;
1068 17 m_functions.vkUpdateDescriptorSets(m_device, 1, &write, 0, nullptr);
1069 17 });
1070 233727 return m_globals;
1071 }
1072
1073 namespace {
1074
1075 608 VkSamplerAddressMode VulkanAddressMode(uint32_t wrap) {
1076
4/4
✓ Branch 0 taken 152 times.
✓ Branch 1 taken 152 times.
✓ Branch 2 taken 152 times.
✓ Branch 3 taken 152 times.
 608 switch (TextureWrap(wrap)) {
1077 case TextureWrap::Repeat:
1078 return VK_SAMPLER_ADDRESS_MODE_REPEAT;
1079 152 case TextureWrap::Mirror:
1080 152 return VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT;
1081 152 case TextureWrap::Clamp:
1082 152 return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
1083 152 default:
1084 152 return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
1085 }
1086 }
1087
1088 28 VkFormat VulkanFormat(const TextureDesc& desc) {
1089 28 const bool encoded = desc.colorSpace == TexelColorSpace::Srgb;
1090 56 const auto pick = [&](VkFormat linear, VkFormat srgb) {
1091
1/4
✗ Branch 0 not taken.
✓ Branch 1 taken 3 times.
✗ Branch 2 not taken.
✗ Branch 3 not taken.
 3 if (encoded && srgb == VK_FORMAT_UNDEFINED) {
1092
0/1
✗ Branch 2 not taken.
 ✗ throw Error("texture \"{}\" is sRGB in a format with no sRGB variant", desc.label);
1093 }
1094
1/2
✓ Branch 0 taken 3 times.
✗ Branch 1 not taken.
 6 return encoded ? srgb : linear;
1095 28 };
1096
2/17
✗ Branch 0 not taken.
✗ Branch 1 not taken.
✓ Branch 2 taken 25 times.
✗ Branch 3 not taken.
✗ Branch 4 not taken.
✗ Branch 5 not taken.
✗ Branch 6 not taken.
✗ Branch 7 not taken.
✓ Branch 8 taken 3 times.
✗ Branch 9 not taken.
✗ Branch 10 not taken.
✗ Branch 11 not taken.
✗ Branch 12 not taken.
✗ Branch 13 not taken.
✗ Branch 14 not taken.
✗ Branch 15 not taken.
✗ Branch 16 not taken.
 28 switch (desc.format) {
1097 ✗ case TexelFormat::R8:
1098
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ return pick(VK_FORMAT_R8_UNORM, VK_FORMAT_R8_SRGB);
1099 ✗ case TexelFormat::Rg8:
1100
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ return pick(VK_FORMAT_R8G8_UNORM, VK_FORMAT_R8G8_SRGB);
1101 25 case TexelFormat::Rgba8:
1102
2/2
✓ Branch 0 taken 7 times.
✓ Branch 1 taken 18 times.
 25 return pick(VK_FORMAT_R8G8B8A8_UNORM, VK_FORMAT_R8G8B8A8_SRGB);
1103 ✗ case TexelFormat::R16:
1104 ✗ return pick(VK_FORMAT_R16_UNORM, VK_FORMAT_UNDEFINED);
1105 ✗ case TexelFormat::Rg16:
1106 ✗ return pick(VK_FORMAT_R16G16_UNORM, VK_FORMAT_UNDEFINED);
1107 ✗ case TexelFormat::Rgba16:
1108 ✗ return pick(VK_FORMAT_R16G16B16A16_UNORM, VK_FORMAT_UNDEFINED);
1109 ✗ case TexelFormat::R16f:
1110 ✗ return pick(VK_FORMAT_R16_SFLOAT, VK_FORMAT_UNDEFINED);
1111 ✗ case TexelFormat::Rg16f:
1112 ✗ return pick(VK_FORMAT_R16G16_SFLOAT, VK_FORMAT_UNDEFINED);
1113 3 case TexelFormat::Rgba16f:
1114 3 return pick(VK_FORMAT_R16G16B16A16_SFLOAT, VK_FORMAT_UNDEFINED);
1115 ✗ case TexelFormat::R32f:
1116 ✗ return pick(VK_FORMAT_R32_SFLOAT, VK_FORMAT_UNDEFINED);
1117 ✗ case TexelFormat::Rg32f:
1118 ✗ return pick(VK_FORMAT_R32G32_SFLOAT, VK_FORMAT_UNDEFINED);
1119 ✗ case TexelFormat::Rgba32f:
1120 ✗ return pick(VK_FORMAT_R32G32B32A32_SFLOAT, VK_FORMAT_UNDEFINED);
1121 ✗ case TexelFormat::Bc1:
1122
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ return pick(VK_FORMAT_BC1_RGBA_UNORM_BLOCK, VK_FORMAT_BC1_RGBA_SRGB_BLOCK);
1123 ✗ case TexelFormat::Bc3:
1124
0/2
✗ Branch 0 not taken.
✗ Branch 1 not taken.
 ✗ return pick(VK_FORMAT_BC3_UNORM_BLOCK, VK_FORMAT_BC3_SRGB_BLOCK);
1125 ✗ case TexelFormat::Bc4:
1126 ✗ return pick(VK_FORMAT_BC4_UNORM_BLOCK, VK_FORMAT_UNDEFINED);
1127 ✗ case TexelFormat::Bc5:
1128 ✗ return pick(VK_FORMAT_BC5_UNORM_BLOCK, VK_FORMAT_UNDEFINED);
1129 }
1130 return VK_FORMAT_UNDEFINED;
1131 }
1132
1133 }
1134
1135 19 void VulkanDevice::CreateSamplers() {
1136 19 const bool anisotropic = Adapter().features.anisotropicFiltering;
1137
2/2
✓ Branch 0 taken 304 times.
✓ Branch 1 taken 19 times.
 323 for (uint32_t wrap = 0; wrap < m_samplers.size(); ++wrap) {
1138 304 VkSamplerCreateInfo info{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO};
1139 304 info.magFilter = VK_FILTER_LINEAR;
1140 304 info.minFilter = VK_FILTER_LINEAR;
1141 304 info.mipmapMode = VK_SAMPLER_MIPMAP_MODE_LINEAR;
1142
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 304 times.
 304 info.anisotropyEnable = anisotropic ? VK_TRUE : VK_FALSE;
1143
2/4
✓ Branch 0 taken 304 times.
✗ Branch 1 not taken.
✗ Branch 2 not taken.
✓ Branch 3 taken 304 times.
 304 info.maxAnisotropy = anisotropic ? std::min(16.0f, Adapter().limits.maxAnisotropy) : 1.0f;
1144 304 info.addressModeU = VulkanAddressMode(wrap >> kTextureWrapBits);
1145 304 info.addressModeV = VulkanAddressMode(wrap & ((1u << kTextureWrapBits) - 1));
1146 304 info.addressModeW = VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
1147 304 info.maxLod = VK_LOD_CLAMP_NONE;
1148 304 info.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
1149 304 Check(m_functions.vkCreateSampler(m_device, &info, nullptr, &m_samplers[wrap]),
1150 "vkCreateSampler");
1151 }
1152 19 }
1153
1154 28 uint32_t VulkanDevice::AcquireTextureSlot(VkImageView view) {
1155 28 const std::lock_guard lock(m_textureMutex);
1156 28 uint32_t slot = 0;
1157
2/2
✓ Branch 0 taken 19 times.
✓ Branch 1 taken 9 times.
 28 if (!m_freeTextureSlots.empty()) {
1158 19 slot = m_freeTextureSlots.back();
1159 19 m_freeTextureSlots.pop_back();
1160
1/2
✓ Branch 0 taken 9 times.
✗ Branch 1 not taken.
 9 } else if (m_textureSlots < Adapter().limits.maxTextures) {
1161 9 slot = m_textureSlots++;
1162 } else {
1163 ✗ throw Error("{} holds at most {} textures at once", Adapter().alias,
1164
0/1
✗ Branch 2 not taken.
 ✗ Adapter().limits.maxTextures);
1165 }
1166
1167 28 VkDescriptorImageInfo image{};
1168 28 image.imageView = view;
1169 28 image.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
1170 28 VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET};
1171
1/1
✓ Branch 1 taken 28 times.
 28 write.dstSet = Globals();
1172 28 write.dstBinding = kTextureBinding;
1173 28 write.dstArrayElement = slot;
1174 28 write.descriptorCount = 1;
1175 28 write.descriptorType = VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE;
1176 28 write.pImageInfo = &image;
1177
1/1
✓ Branch 1 taken 28 times.
 28 m_functions.vkUpdateDescriptorSets(m_device, 1, &write, 0, nullptr);
1178 28 return slot;
1179 28 }
1180
1181 28 void VulkanDevice::ReleaseTextureSlot(uint32_t slot) {
1182 28 const std::lock_guard lock(m_textureMutex);
1183
1/1
✓ Branch 1 taken 28 times.
 28 m_freeTextureSlots.push_back(slot);
1184 28 }
1185
1186 28 Ref<Texture> VulkanDevice::NewTexture(const TextureDesc& desc) {
1187
1/1
✓ Branch 2 taken 28 times.
 28 return Ref<Texture>(new VulkanTexture(*this, desc));
1188 }
1189
1190 28 VulkanTexture::VulkanTexture(VulkanDevice& owner, const TextureDesc& desc)
1191
1/1
✓ Branch 2 taken 28 times.
 28 : Texture(owner, desc), m_owner(owner) {
1192
1/1
✓ Branch 1 taken 28 times.
 28 const VkFormat format = VulkanFormat(desc);
1193 28 VkFormatProperties properties{};
1194
1/1
✓ Branch 1 taken 28 times.
 28 vkGetPhysicalDeviceFormatProperties(owner.PhysicalDevice(), format, &properties);
1195 28 constexpr VkFormatFeatureFlags kNeeded = VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT |
1196 VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_LINEAR_BIT |
1197 VK_FORMAT_FEATURE_TRANSFER_DST_BIT;
1198
1/2
✗ Branch 0 not taken.
✓ Branch 1 taken 28 times.
 28 if ((properties.optimalTilingFeatures & kNeeded) != kNeeded) {
1199 ✗ throw UnsupportedError("{} can't filter the format of texture \"{}\"",
1200
0/1
✗ Branch 2 not taken.
 ✗ owner.Adapter().alias, desc.label);
1201 }
1202
1203 28 VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO};
1204 28 info.imageType = VK_IMAGE_TYPE_2D;
1205 28 info.format = format;
1206 28 info.extent = {desc.width, desc.height, 1};
1207 28 info.mipLevels = desc.levels;
1208 28 info.arrayLayers = 1;
1209 28 info.samples = VK_SAMPLE_COUNT_1_BIT;
1210 28 info.tiling = VK_IMAGE_TILING_OPTIMAL;
1211 28 info.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;
1212 28 info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
1213 28 info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
1214 28 VmaAllocationCreateInfo allocation{};
1215 28 allocation.usage = VMA_MEMORY_USAGE_AUTO_PREFER_DEVICE;
1216
2/2
✓ Branch 1 taken 28 times.
✓ Branch 4 taken 28 times.
 28 Check(vmaCreateImage(owner.Allocator(), &info, &allocation, &m_image, &m_allocation, nullptr),
1217 "vmaCreateImage");
1218
1219 28 try {
1220
1/1
✓ Branch 1 taken 28 times.
 28 vmaSetAllocationName(owner.Allocator(), m_allocation, desc.label.c_str());
1221
1/1
✓ Branch 1 taken 28 times.
 28 owner.Name(VK_OBJECT_TYPE_IMAGE, uint64_t(m_image), desc.label);
1222 28 VkImageViewCreateInfo view{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO};
1223 28 view.image = m_image;
1224 28 view.viewType = VK_IMAGE_VIEW_TYPE_2D;
1225 28 view.format = format;
1226 28 view.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, desc.levels, 0, 1};
1227
2/2
✓ Branch 1 taken 28 times.
✓ Branch 4 taken 28 times.
 28 Check(owner.Functions().vkCreateImageView(owner.Handle(), &view, nullptr, &m_view),
1228 "vkCreateImageView");
1229
1/1
✓ Branch 1 taken 28 times.
 28 m_slot = owner.AcquireTextureSlot(m_view);
1230 ✗ } catch (...) {
1231 ✗ Destroy();
1232 ✗ throw;
1233 ✗ }
1234 28 m_handle = TextureHandle(*m_slot, desc.wrapS, desc.wrapT);
1235 28 }
1236
1237 56 VulkanTexture::~VulkanTexture() {
1238 28 Destroy();
1239 56 }
1240
1241 28 void VulkanTexture::Destroy() noexcept {
1242
1/2
✓ Branch 0 taken 28 times.
✗ Branch 1 not taken.
 28 if (m_slot) {
1243 28 m_owner.ReleaseTextureSlot(*m_slot);
1244
1/2
✓ Branch 0 taken 28 times.
✗ Branch 1 not taken.
 28 m_slot.reset();
1245 }
1246 28 m_owner.Functions().vkDestroyImageView(m_owner.Handle(), m_view, nullptr);
1247 28 vmaDestroyImage(m_owner.Allocator(), m_image, m_allocation);
1248 28 m_view = VK_NULL_HANDLE;
1249 28 m_image = VK_NULL_HANDLE;
1250 28 m_allocation = VK_NULL_HANDLE;
1251 28 }
1252
1253 1119 std::unique_ptr<Kernel> VulkanDevice::NewKernel() {
1254 1119 return std::make_unique<VulkanKernel>(*this);
1255 }
1256
1257 }
1258