device/vulkan/device.h
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | #pragma once | ||
| 3 | |||
| 4 | #include <algorithm> | ||
| 5 | #include <array> | ||
| 6 | #include <cstring> | ||
| 7 | #include <memory> | ||
| 8 | #include <mutex> | ||
| 9 | #include <optional> | ||
| 10 | #include <span> | ||
| 11 | #include <vector> | ||
| 12 | |||
| 13 | #include <volk.h> | ||
| 14 | |||
| 15 | #include <vk_mem_alloc.h> | ||
| 16 | |||
| 17 | #include "chrono_batch_render/device/accel.h" | ||
| 18 | #include "chrono_batch_render/device/device.h" | ||
| 19 | #include "chrono_batch_render/device/operation.h" | ||
| 20 | #include "chrono_batch_render/device/program.h" | ||
| 21 | #include "chrono_batch_render/device/texture.h" | ||
| 22 | #include "chrono_batch_render/device/vulkan/api.h" | ||
| 23 | |||
| 24 | namespace chrono::batch_render::device::vulkan { | ||
| 25 | |||
| 26 | class MemoryInterop; | ||
| 27 | |||
| 28 | inline constexpr const char* kRayQueryExtensions[] = { | ||
| 29 | VK_KHR_ACCELERATION_STRUCTURE_EXTENSION_NAME, VK_KHR_RAY_QUERY_EXTENSION_NAME, | ||
| 30 | VK_KHR_DEFERRED_HOST_OPERATIONS_EXTENSION_NAME}; | ||
| 31 | |||
| 32 | inline constexpr uint32_t kMaxTextures = 65536; | ||
| 33 | |||
| 34 | struct FeatureChain { | ||
| 35 | VkPhysicalDeviceFeatures2 base{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2}; | ||
| 36 | VkPhysicalDeviceVulkan12Features vulkan12{ | ||
| 37 | VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES}; | ||
| 38 | VkPhysicalDeviceVulkan13Features vulkan13{ | ||
| 39 | VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES}; | ||
| 40 | VkPhysicalDeviceAccelerationStructureFeaturesKHR accelerationStructure{ | ||
| 41 | VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ACCELERATION_STRUCTURE_FEATURES_KHR}; | ||
| 42 | VkPhysicalDeviceRayQueryFeaturesKHR rayQuery{ | ||
| 43 | VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_RAY_QUERY_FEATURES_KHR}; | ||
| 44 | VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomicFloat{ | ||
| 45 | VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT}; | ||
| 46 | VkPhysicalDeviceShaderClockFeaturesKHR clock{ | ||
| 47 | VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_CLOCK_FEATURES_KHR}; | ||
| 48 | VkPhysicalDevicePipelineExecutablePropertiesFeaturesKHR executables{ | ||
| 49 | VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PIPELINE_EXECUTABLE_PROPERTIES_FEATURES_KHR}; | ||
| 50 | |||
| 51 | 62 | template <class Has> void Link(Has has) { | |
| 52 | 62 | base.pNext = &vulkan12; | |
| 53 | 62 | vulkan12.pNext = &vulkan13; | |
| 54 | 62 | void** tail = &vulkan13.pNext; | |
| 55 | 352 | const auto chain = [&tail](auto& features, bool present) { | |
| 56 | 248 | if (present) { | |
| 57 | 126 | *tail = &features; | |
| 58 | 84 | tail = &features.pNext; | |
| 59 | } | ||
| 60 | }; | ||
| 61 | 62 | const bool rayQueries = std::ranges::all_of(kRayQueryExtensions, has); | |
| 62 | 62 | chain(accelerationStructure, rayQueries); | |
| 63 | 42 | chain(rayQuery, rayQueries); | |
| 64 | 104 | chain(atomicFloat, has(VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME)); | |
| 65 | 104 | chain(clock, has(VK_KHR_SHADER_CLOCK_EXTENSION_NAME)); | |
| 66 | 62 | chain(executables, has(VK_KHR_PIPELINE_EXECUTABLE_PROPERTIES_EXTENSION_NAME)); | |
| 67 | 62 | *tail = nullptr; | |
| 68 | 62 | } | |
| 69 | }; | ||
| 70 | |||
| 71 | struct ComputeFamily { | ||
| 72 | uint32_t index; | ||
| 73 | VkQueueFamilyProperties properties; | ||
| 74 | }; | ||
| 75 | |||
| 76 | std::optional<ComputeFamily> ComputeQueueFamily(VkPhysicalDevice device, | ||
| 77 | VkQueueFlags flags = VK_QUEUE_COMPUTE_BIT); | ||
| 78 | |||
| 79 | std::vector<AdapterInfo> ListAdapters(); | ||
| 80 | |||
| 81 | std::array<uint8_t, 16> DeviceUuid(VkPhysicalDevice device); | ||
| 82 | |||
| 83 | class VulkanDevice : public Device { | ||
| 84 | public: | ||
| 85 | VulkanDevice(const AdapterInfo& adapter, const DeviceDesc& desc, Traversal traversal, | ||
| 86 | OwnedInstance instance = {}); | ||
| 87 | ~VulkanDevice() override; | ||
| 88 | |||
| 89 | 2 | VkInstance InstanceHandle() const { | |
| 90 | 2 | return m_instance; | |
| 91 | } | ||
| 92 | |||
| 93 | 1169196 | VkDevice Handle() const { | |
| 94 | 1168049 | return m_device; | |
| 95 | } | ||
| 96 | |||
| 97 | 30 | VkPhysicalDevice PhysicalDevice() const { | |
| 98 | 30 | return m_physicalDevice; | |
| 99 | } | ||
| 100 | |||
| 101 | VkQueue QueueHandle() const { | ||
| 102 | return m_queueHandle; | ||
| 103 | } | ||
| 104 | |||
| 105 | 22 | uint32_t QueueFamily() const { | |
| 106 | 22 | return m_queueFamily; | |
| 107 | } | ||
| 108 | |||
| 109 | ✗ | Queue& DisplayQueue() { | |
| 110 | ✗ | return m_displayQueue ? *m_displayQueue : GetQueue(); | |
| 111 | } | ||
| 112 | |||
| 113 | 2 | bool SharesDisplayQueue() const { | |
| 114 | 2 | return !m_displayQueue; | |
| 115 | } | ||
| 116 | |||
| 117 | 843577 | const VolkDeviceTable& Functions() const { | |
| 118 | 1076289 | return m_functions; | |
| 119 | } | ||
| 120 | |||
| 121 | 233699 | VkPipelineLayout PipelineLayout() const { | |
| 122 | 233699 | return m_pipelineLayout; | |
| 123 | } | ||
| 124 | |||
| 125 | 1117 | VkPipelineCache PipelineCache() const { | |
| 126 | 1117 | return m_pipelineCache; | |
| 127 | } | ||
| 128 | |||
| 129 | 120067 | VmaAllocator Allocator() const { | |
| 130 | 120067 | return m_allocator; | |
| 131 | } | ||
| 132 | |||
| 133 | 20 | bool SharesSemaphores() const { | |
| 134 | 20 | return m_sharesSemaphores; | |
| 135 | } | ||
| 136 | |||
| 137 | 13 | bool SharesExportedMemory() const { | |
| 138 | 13 | return m_sharesMemory; | |
| 139 | } | ||
| 140 | |||
| 141 | 26 | uint32_t VendorId() const { | |
| 142 | 13 | return m_vendorId; | |
| 143 | } | ||
| 144 | |||
| 145 | 11 | bool HasTimestamps() const { | |
| 146 | 11 | return m_timestampMask != 0; | |
| 147 | } | ||
| 148 | |||
| 149 | 2234 | bool CapturesStatistics() const { | |
| 150 | 2234 | return m_capturesStatistics; | |
| 151 | } | ||
| 152 | |||
| 153 | 98368 | bool Labels() const { | |
| 154 | 98368 | return m_labels; | |
| 155 | } | ||
| 156 | |||
| 157 | void Name(VkObjectType type, uint64_t object, const std::string& name) const; | ||
| 158 | bool ToNanoseconds(std::span<uint64_t> timestamps) const; | ||
| 159 | std::optional<std::pair<uint64_t, uint64_t>> CalibratedTimestamps() const; | ||
| 160 | |||
| 161 | struct SpareBuffer { | ||
| 162 | VkBuffer buffer = VK_NULL_HANDLE; | ||
| 163 | VmaAllocation allocation = VK_NULL_HANDLE; | ||
| 164 | void* external = nullptr; | ||
| 165 | uint64_t externalAddress = 0; | ||
| 166 | uint64_t bytes = 0; | ||
| 167 | BufferUsage usage = BufferUsage::None; | ||
| 168 | }; | ||
| 169 | |||
| 170 | uint64_t BufferAddress(VkBuffer buffer) const; | ||
| 171 | MemoryInterop* GetMemoryInterop(); | ||
| 172 | bool KeepSpare(const SpareBuffer& spare); | ||
| 173 | std::optional<DLDevice> SharedDevice() override; | ||
| 174 | Ref<Buffer> Import(Buffer& foreign) override; | ||
| 175 | Ref<Buffer> ImportMemory(const VulkanDevice& owner, VkBuffer buffer, VmaAllocation allocation, | ||
| 176 | const BufferDesc& desc); | ||
| 177 | |||
| 178 | VkDescriptorSet Globals(); | ||
| 179 | uint32_t AcquireTextureSlot(VkImageView view); | ||
| 180 | void ReleaseTextureSlot(uint32_t slot); | ||
| 181 | |||
| 182 | protected: | ||
| 183 | AccelSizes HardwareBuildSizes(const AccelInput& input) const override; | ||
| 184 | Ref<Accel> NewAccel(Ref<Buffer> storage, AccelLevel level, AccelSizes sizes, | ||
| 185 | std::string label) override; | ||
| 186 | void ExternalCopy(BufferRange dst, uint64_t source, Stream producer) override; | ||
| 187 | Stream CopyStream() override; | ||
| 188 | Ref<Buffer> NewBuffer(const BufferDesc& desc) override; | ||
| 189 | std::vector<MemoryHeap> Heaps() const override; | ||
| 190 | Ref<Texture> NewTexture(const TextureDesc& desc) override; | ||
| 191 | std::unique_ptr<Kernel> NewKernel() override; | ||
| 192 | |||
| 193 | private: | ||
| 194 | void CreateDescriptors(bool textures); | ||
| 195 | void Destroy(); | ||
| 196 | VkBufferUsageFlags UsageFlags(BufferUsage usage) const; | ||
| 197 | VmaPool InteropPool(); | ||
| 198 | std::optional<SpareBuffer> TakeSpare(uint64_t bytes, BufferUsage usage); | ||
| 199 | void ReleaseSpares(); | ||
| 200 | void CreateSamplers(); | ||
| 201 | |||
| 202 | OwnedInstance m_ownedInstance; | ||
| 203 | VkInstance m_instance = VK_NULL_HANDLE; | ||
| 204 | VkPhysicalDevice m_physicalDevice = VK_NULL_HANDLE; | ||
| 205 | VkDevice m_device = VK_NULL_HANDLE; | ||
| 206 | VkQueue m_queueHandle = VK_NULL_HANDLE; | ||
| 207 | uint32_t m_queueFamily = 0; | ||
| 208 | std::unique_ptr<Queue> m_displayQueue; | ||
| 209 | VolkDeviceTable m_functions{}; | ||
| 210 | VmaAllocator m_allocator = VK_NULL_HANDLE; | ||
| 211 | VkPipelineCache m_pipelineCache = VK_NULL_HANDLE; | ||
| 212 | VkDescriptorSetLayout m_globalsLayout = VK_NULL_HANDLE; | ||
| 213 | VkPipelineLayout m_pipelineLayout = VK_NULL_HANDLE; | ||
| 214 | VkDescriptorPool m_descriptorPool = VK_NULL_HANDLE; | ||
| 215 | PFN_vkGetCalibratedTimestampsKHR m_calibratedTimestamps = nullptr; | ||
| 216 | double m_timestampPeriod = 0; | ||
| 217 | uint64_t m_timestampMask = 0; | ||
| 218 | bool m_capturesStatistics = false; | ||
| 219 | bool m_labels = false; | ||
| 220 | bool m_sharesSemaphores = false; | ||
| 221 | bool m_sharesMemory = false; | ||
| 222 | VkExportMemoryAllocateInfo m_exportInfo{VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO}; | ||
| 223 | VmaPool m_interopPool = VK_NULL_HANDLE; | ||
| 224 | std::mutex m_interopMutex; | ||
| 225 | bool m_interopTried = false; | ||
| 226 | std::unique_ptr<MemoryInterop> m_memoryInterop; | ||
| 227 | std::mutex m_spareMutex; | ||
| 228 | std::vector<SpareBuffer> m_spares; | ||
| 229 | uint64_t m_spareBytes = 0; | ||
| 230 | bool m_sparesClosed = false; | ||
| 231 | uint32_t m_vendorId = 0; | ||
| 232 | VkDescriptorSet m_globals = VK_NULL_HANDLE; | ||
| 233 | std::once_flag m_globalsWritten; | ||
| 234 | std::array<VkSampler, kTextureSamplers> m_samplers{}; | ||
| 235 | std::mutex m_textureMutex; | ||
| 236 | std::vector<uint32_t> m_freeTextureSlots; | ||
| 237 | uint32_t m_textureSlots = 0; | ||
| 238 | }; | ||
| 239 | |||
| 240 | struct VulkanBuild { | ||
| 241 | VkAccelerationStructureGeometryKHR geometry{ | ||
| 242 | VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_GEOMETRY_KHR}; | ||
| 243 | VkAccelerationStructureBuildGeometryInfoKHR info{ | ||
| 244 | VK_STRUCTURE_TYPE_ACCELERATION_STRUCTURE_BUILD_GEOMETRY_INFO_KHR}; | ||
| 245 | uint32_t primitives = 0; | ||
| 246 | |||
| 247 | void Describe(const AccelInput& input); | ||
| 248 | }; | ||
| 249 | |||
| 250 | class VulkanAccel : public Accel { | ||
| 251 | public: | ||
| 252 | VulkanAccel(VulkanDevice& owner, Ref<Buffer> storage, AccelLevel level, AccelSizes sizes, | ||
| 253 | std::string label); | ||
| 254 | ~VulkanAccel() override; | ||
| 255 | |||
| 256 | ✗ | VkAccelerationStructureKHR Handle() const { | |
| 257 | ✗ | return m_handle; | |
| 258 | } | ||
| 259 | |||
| 260 | private: | ||
| 261 | VulkanDevice& m_owner; | ||
| 262 | VkAccelerationStructureKHR m_handle = VK_NULL_HANDLE; | ||
| 263 | }; | ||
| 264 | |||
| 265 | class VulkanBuffer : public Buffer { | ||
| 266 | public: | ||
| 267 | 119983 | VulkanBuffer(VulkanDevice& owner, const BufferDesc& desc, VkBuffer buffer, | |
| 268 | VmaAllocation allocation, void* mapped, void* external = nullptr, | ||
| 269 | uint64_t externalAddress = 0) | ||
| 270 | 239966 | : Buffer(owner, desc), m_owner(owner), m_buffer(buffer), m_allocation(allocation), | |
| 271 | 119983 | m_external(external), m_externalAddress(externalAddress) { | |
| 272 | 119983 | m_address = owner.BufferAddress(buffer); | |
| 273 | 119983 | m_mapped = mapped; | |
| 274 | 119983 | } | |
| 275 | |||
| 276 | ~VulkanBuffer() override; | ||
| 277 | |||
| 278 | 446348 | VkBuffer Handle() const { | |
| 279 | 446348 | return m_buffer; | |
| 280 | } | ||
| 281 | |||
| 282 | ExternalAddress External(DLDeviceType api) override; | ||
| 283 | |||
| 284 | ✗ | VulkanDevice& Owner() const { | |
| 285 | ✗ | return m_owner; | |
| 286 | } | ||
| 287 | |||
| 288 | bool exportable = false; | ||
| 289 | |||
| 290 | Ref<Buffer> SharedWith(VulkanDevice& reader); | ||
| 291 | |||
| 292 | protected: | ||
| 293 | VulkanDevice& m_owner; | ||
| 294 | VkBuffer m_buffer; | ||
| 295 | VmaAllocation m_allocation = VK_NULL_HANDLE; | ||
| 296 | std::mutex m_externalMutex; | ||
| 297 | void* m_external = nullptr; | ||
| 298 | uint64_t m_externalAddress = 0; | ||
| 299 | std::vector<std::pair<VulkanDevice*, Ref<Buffer>>> m_shared; | ||
| 300 | }; | ||
| 301 | |||
| 302 | class ImportedVulkanBuffer : public VulkanBuffer { | ||
| 303 | public: | ||
| 304 | ✗ | ImportedVulkanBuffer(VulkanDevice& owner, const BufferDesc& desc, VkBuffer buffer, | |
| 305 | VkDeviceMemory memory) | ||
| 306 | ✗ | : VulkanBuffer(owner, desc, buffer, VK_NULL_HANDLE, nullptr), m_memory(memory) {} | |
| 307 | |||
| 308 | ~ImportedVulkanBuffer() override; | ||
| 309 | |||
| 310 | private: | ||
| 311 | VkDeviceMemory m_memory; | ||
| 312 | }; | ||
| 313 | |||
| 314 | class VulkanKernel : public Kernel { | ||
| 315 | public: | ||
| 316 | 1119 | explicit VulkanKernel(VulkanDevice& owner) : m_owner(owner) {} | |
| 317 | |||
| 318 | ~VulkanKernel() override; | ||
| 319 | |||
| 320 | 232582 | VkPipeline Pipeline() const { | |
| 321 | 232582 | return m_pipeline; | |
| 322 | } | ||
| 323 | |||
| 324 | protected: | ||
| 325 | KernelStats Load(const KernelCode& code, const DiskCache& cache, | ||
| 326 | const DeviceGlobals& globals) override; | ||
| 327 | |||
| 328 | private: | ||
| 329 | VulkanDevice& m_owner; | ||
| 330 | VkPipeline m_pipeline = VK_NULL_HANDLE; | ||
| 331 | }; | ||
| 332 | |||
| 333 | class VulkanTexture : public Texture { | ||
| 334 | public: | ||
| 335 | VulkanTexture(VulkanDevice& owner, const TextureDesc& desc); | ||
| 336 | ~VulkanTexture() override; | ||
| 337 | |||
| 338 | 130 | VkImage Image() const { | |
| 339 | 130 | return m_image; | |
| 340 | } | ||
| 341 | |||
| 342 | 30 | uint64_t KernelHandle(TextureWrap wrapS, TextureWrap wrapT) override { | |
| 343 | 30 | return uint64_t(1) << 32 | TextureHandle(*m_slot, wrapS, wrapT); | |
| 344 | } | ||
| 345 | |||
| 346 | private: | ||
| 347 | void Destroy() noexcept; | ||
| 348 | |||
| 349 | VulkanDevice& m_owner; | ||
| 350 | VkImage m_image = VK_NULL_HANDLE; | ||
| 351 | VmaAllocation m_allocation = VK_NULL_HANDLE; | ||
| 352 | VkImageView m_view = VK_NULL_HANDLE; | ||
| 353 | std::optional<uint32_t> m_slot; | ||
| 354 | }; | ||
| 355 | |||
| 356 | std::unique_ptr<Queue> CreateQueue(VulkanDevice& device, VkQueue handle, bool runsKernels); | ||
| 357 | |||
| 358 | struct NativeSubmit; | ||
| 359 | |||
| 360 | Operation SubmitNative(Queue& queue, const NativeSubmit& submit); | ||
| 361 | |||
| 362 | VkResult PresentNative(Queue& queue, const VkPresentInfoKHR& present); | ||
| 363 | |||
| 364 | VkQueue NativeQueue(Queue& queue); | ||
| 365 | |||
| 366 | } | ||
| 367 |