device/device.h
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | #pragma once | ||
| 3 | |||
| 4 | #include <cstdint> | ||
| 5 | #include <filesystem> | ||
| 6 | #include <memory> | ||
| 7 | #include <optional> | ||
| 8 | #include <string> | ||
| 9 | #include <variant> | ||
| 10 | #include <vector> | ||
| 11 | |||
| 12 | #include <dlpack/dlpack.h> | ||
| 13 | |||
| 14 | #include "chrono_batch_render/device/adapter.h" | ||
| 15 | #include "chrono_batch_render/device/operation.h" | ||
| 16 | #include "chrono_batch_render/device/ref.h" | ||
| 17 | #include "chrono_batch_render/device/stream.h" | ||
| 18 | #include "chrono_batch_render/device/tensor.h" | ||
| 19 | #include "chrono_batch_render/export.h" | ||
| 20 | #include "chrono_batch_render/util/span.h" | ||
| 21 | |||
| 22 | namespace chrono::batch_render { | ||
| 23 | struct DeviceGlobals; | ||
| 24 | } | ||
| 25 | |||
| 26 | namespace chrono::batch_render::device { | ||
| 27 | |||
| 28 | 3 | struct BuildProgress { | |
| 29 | uint64_t queued = 0; | ||
| 30 | uint64_t finished = 0; | ||
| 31 | uint64_t cached = 0; | ||
| 32 | std::vector<std::string> building; | ||
| 33 | |||
| 34 | 3 | bool Done() const { | |
| 35 | 3 | return finished >= queued; | |
| 36 | } | ||
| 37 | }; | ||
| 38 | |||
| 39 | CHRONO_BATCH_RENDER_API BuildProgress CurrentBuilds(); | ||
| 40 | |||
| 41 | class Accel; | ||
| 42 | class Buffer; | ||
| 43 | class BufferRange; | ||
| 44 | class DebugRecords; | ||
| 45 | class Kernel; | ||
| 46 | class Program; | ||
| 47 | class Queue; | ||
| 48 | class Resource; | ||
| 49 | class Texture; | ||
| 50 | struct AccelBoxes; | ||
| 51 | struct AccelInstances; | ||
| 52 | struct AccelSizes; | ||
| 53 | struct AccelTriangles; | ||
| 54 | struct BufferDesc; | ||
| 55 | struct ProgramDesc; | ||
| 56 | struct TextureDesc; | ||
| 57 | enum class AccelLevel : uint8_t; | ||
| 58 | |||
| 59 | using AccelInput = std::variant<AccelTriangles, AccelBoxes, AccelInstances>; | ||
| 60 | |||
| 61 | namespace vulkan { | ||
| 62 | struct Creator; | ||
| 63 | } | ||
| 64 | |||
| 65 | struct DeviceDesc { | ||
| 66 | std::optional<Backend> backend; | ||
| 67 | std::string adapter; | ||
| 68 | Traversal traversal = Traversal::Auto; | ||
| 69 | Sampling sampling = Sampling::Auto; | ||
| 70 | bool validation = false; | ||
| 71 | bool kernelDebugInfo = false; | ||
| 72 | bool profiling = true; | ||
| 73 | bool synchronous = false; | ||
| 74 | bool presentation = false; | ||
| 75 | std::filesystem::path cacheDirectory; | ||
| 76 | std::shared_ptr<const vulkan::Creator> creator; | ||
| 77 | }; | ||
| 78 | |||
| 79 | 41455 | struct MemoryUse { | |
| 80 | std::string label; | ||
| 81 | uint32_t buffers = 0; | ||
| 82 | uint32_t textures = 0; | ||
| 83 | uint64_t bytes = 0; | ||
| 84 | }; | ||
| 85 | |||
| 86 | struct MemoryHeap { | ||
| 87 | uint64_t usage = 0; | ||
| 88 | uint64_t budget = 0; | ||
| 89 | uint64_t size = 0; | ||
| 90 | bool deviceLocal = false; | ||
| 91 | }; | ||
| 92 | |||
| 93 | 10 | struct MemoryStats { | |
| 94 | std::vector<MemoryUse> uses; | ||
| 95 | std::vector<MemoryHeap> heaps; | ||
| 96 | uint64_t deviceBytes = 0; | ||
| 97 | uint64_t textureBytes = 0; | ||
| 98 | uint64_t uploadBytes = 0; | ||
| 99 | uint64_t readbackBytes = 0; | ||
| 100 | uint64_t sharedBytes = 0; | ||
| 101 | uint64_t reservedBytes = 0; | ||
| 102 | }; | ||
| 103 | |||
| 104 | struct TrafficStats { | ||
| 105 | uint64_t uploadBytes = 0; | ||
| 106 | uint64_t readbackBytes = 0; | ||
| 107 | uint64_t inlineBytes = 0; | ||
| 108 | uint64_t copyBytes = 0; | ||
| 109 | uint64_t submissions = 0; | ||
| 110 | uint64_t hostWaits = 0; | ||
| 111 | |||
| 112 | 344376 | TrafficStats& operator+=(const TrafficStats& other) { | |
| 113 | 344376 | uploadBytes += other.uploadBytes; | |
| 114 | 344376 | readbackBytes += other.readbackBytes; | |
| 115 | 344376 | inlineBytes += other.inlineBytes; | |
| 116 | 344376 | copyBytes += other.copyBytes; | |
| 117 | 344376 | submissions += other.submissions; | |
| 118 | 344376 | hostWaits += other.hostWaits; | |
| 119 | 344376 | return *this; | |
| 120 | } | ||
| 121 | }; | ||
| 122 | |||
| 123 | 1 | inline TrafficStats operator-(const TrafficStats& a, const TrafficStats& b) { | |
| 124 | 1 | TrafficStats result; | |
| 125 | 1 | result.uploadBytes = a.uploadBytes - b.uploadBytes; | |
| 126 | 1 | result.readbackBytes = a.readbackBytes - b.readbackBytes; | |
| 127 | 1 | result.inlineBytes = a.inlineBytes - b.inlineBytes; | |
| 128 | 1 | result.copyBytes = a.copyBytes - b.copyBytes; | |
| 129 | 1 | result.submissions = a.submissions - b.submissions; | |
| 130 | 1 | result.hostWaits = a.hostWaits - b.hostWaits; | |
| 131 | 1 | return result; | |
| 132 | } | ||
| 133 | |||
| 134 | class CHRONO_BATCH_RENDER_API Device : public RefCounted { | ||
| 135 | public: | ||
| 136 | static Ref<Device> Create(const DeviceDesc& desc = {}); | ||
| 137 | |||
| 138 | ~Device() override; | ||
| 139 | |||
| 140 | 40 | const AdapterInfo& Adapter() const { | |
| 141 | 2788 | return m_adapter; | |
| 142 | } | ||
| 143 | |||
| 144 | 120678 | Traversal GetTraversal() const { | |
| 145 | 120678 | return m_traversal; | |
| 146 | } | ||
| 147 | |||
| 148 | 1173 | Sampling GetSampling() const { | |
| 149 | 1173 | return m_sampling; | |
| 150 | } | ||
| 151 | |||
| 152 | ✗ | bool Presents() const { | |
| 153 | ✗ | return m_presents; | |
| 154 | } | ||
| 155 | |||
| 156 | const std::filesystem::path& CacheDirectory() const { | ||
| 157 | 1 | return m_cacheDirectory; | |
| 158 | } | ||
| 159 | |||
| 160 | 1199 | const std::filesystem::path& KernelCacheDirectory() const { | |
| 161 | 1199 | return m_kernelCacheDirectory; | |
| 162 | } | ||
| 163 | |||
| 164 | Ref<Buffer> CreateBuffer(const BufferDesc& desc); | ||
| 165 | Ref<Buffer> Borrow(DLDevice source, uint64_t address, uint64_t bytes, | ||
| 166 | std::string label = "borrowed"); | ||
| 167 | Ref<Buffer> CreateBuffer(BufferDesc desc, Span<const std::byte> data); | ||
| 168 | template <class T> Ref<Buffer> CreateBuffer(BufferDesc desc, Span<const T> values); | ||
| 169 | #ifdef __cpp_lib_span | ||
| 170 | template <class T> Ref<Buffer> CreateBuffer(BufferDesc desc, std::span<const T> values); | ||
| 171 | #endif | ||
| 172 | template <class T> Ref<Buffer> CreateBuffer(BufferDesc desc, const std::vector<T>& values); | ||
| 173 | |||
| 174 | AccelSizes AccelBuildSizes(const AccelInput& input) const; | ||
| 175 | |||
| 176 | Ref<Accel> CreateAccel(const AccelInput& input, std::string label = "accel"); | ||
| 177 | |||
| 178 | Ref<Texture> CreateTexture(const TextureDesc& desc); | ||
| 179 | |||
| 180 | 3 | Tensor CreateTensor(DType type, std::vector<int64_t> shape, std::string label = "tensor"); | |
| 181 | Tensor CreateSharedTensor(DType type, std::vector<int64_t> shape, | ||
| 182 | std::string label = "shared tensor"); | ||
| 183 | |||
| 184 | Operation Copy(const Tensor& src, const Tensor& dst, const Operation& after = {}); | ||
| 185 | Tensor Import(const DLTensor& source, Stream producer = Stream::None(), | ||
| 186 | 3 | std::string label = "imported"); | |
| 187 | Operation Follow(Stream producer); | ||
| 188 | |||
| 189 | Ref<Program> CreateProgram(const ProgramDesc& desc); | ||
| 190 | |||
| 191 | MemoryStats MemoryUsage() const; | ||
| 192 | |||
| 193 | TrafficStats Traffic() const; | ||
| 194 | |||
| 195 | 15666041 | Queue& GetQueue() { | |
| 196 | 15666039 | return *m_queue; | |
| 197 | } | ||
| 198 | |||
| 199 | void Require(const Resource& resource, const char* what) const; | ||
| 200 | |||
| 201 | virtual std::optional<DLDevice> SharedDevice(); | ||
| 202 | virtual Ref<Buffer> Import(Buffer& foreign); | ||
| 203 | |||
| 204 | bool SharesMemory(DLDeviceType api) { | ||
| 205 | const std::optional<DLDevice> shared = SharedDevice(); | ||
| 206 | return shared && shared->device_type == api; | ||
| 207 | } | ||
| 208 | |||
| 209 | bool ReadsDirectly(DLDevice source) const; | ||
| 210 | void CopyIn(BufferRange dst, uint64_t source, Stream producer); | ||
| 211 | Ref<Buffer> Stage(uint64_t source, uint64_t bytes, Stream producer, const std::string& label); | ||
| 212 | Ref<Buffer> StageOnStream(uint64_t source, uint64_t bytes, Stream producer, | ||
| 213 | const std::string& label); | ||
| 214 | |||
| 215 | protected: | ||
| 216 | Device(AdapterInfo adapter, const DeviceDesc& desc, Traversal traversal); | ||
| 217 | |||
| 218 | virtual Ref<Buffer> NewBuffer(const BufferDesc& desc) = 0; | ||
| 219 | virtual void ExternalCopy(BufferRange dst, uint64_t source, Stream producer); | ||
| 220 | virtual Stream CopyStream(); | ||
| 221 | |||
| 222 | virtual AccelSizes HardwareBuildSizes(const AccelInput& input) const; | ||
| 223 | |||
| 224 | virtual Ref<Accel> NewAccel(Ref<Buffer> storage, AccelLevel level, AccelSizes sizes, | ||
| 225 | std::string label); | ||
| 226 | |||
| 227 | virtual Ref<Texture> NewTexture(const TextureDesc& desc); | ||
| 228 | |||
| 229 | virtual std::unique_ptr<Kernel> NewKernel() = 0; | ||
| 230 | |||
| 231 | virtual std::vector<MemoryHeap> Heaps() const; | ||
| 232 | |||
| 233 | 223241 | DebugRecords& Debug() { | |
| 234 | 223241 | return *m_debug; | |
| 235 | } | ||
| 236 | |||
| 237 | static void CloseQueue(Queue& queue) noexcept; | ||
| 238 | |||
| 239 | DeviceGlobals Globals() const; | ||
| 240 | |||
| 241 | 17 | Buffer& GlobalsBuffer() const { | |
| 242 | 17 | return *m_globals; | |
| 243 | } | ||
| 244 | |||
| 245 | std::unique_ptr<Queue> m_queue; | ||
| 246 | |||
| 247 | private: | ||
| 248 | friend class Buffer; | ||
| 249 | friend class Texture; | ||
| 250 | friend class CommandEncoder; | ||
| 251 | friend class Program; | ||
| 252 | friend class Queue; | ||
| 253 | friend class Resource; | ||
| 254 | struct Counters; | ||
| 255 | struct AccelSizeCache; | ||
| 256 | struct ProgramCache; | ||
| 257 | |||
| 258 | void CountTraffic(const TrafficStats& traffic); | ||
| 259 | |||
| 260 | void CreateGlobals(); | ||
| 261 | |||
| 262 | void LastReleased() noexcept override; | ||
| 263 | |||
| 264 | void Retire(Resource& resource) noexcept; | ||
| 265 | |||
| 266 | static void Adopt(Resource& resource); | ||
| 267 | |||
| 268 | void CountMemory(Buffer& buffer, int sign); | ||
| 269 | void CountMemory(Texture& texture, int sign); | ||
| 270 | |||
| 271 | AdapterInfo m_adapter; | ||
| 272 | Traversal m_traversal; | ||
| 273 | Sampling m_sampling; | ||
| 274 | std::filesystem::path m_cacheDirectory; | ||
| 275 | std::filesystem::path m_kernelCacheDirectory; | ||
| 276 | bool m_profiling; | ||
| 277 | bool m_synchronous; | ||
| 278 | bool m_kernelDebugInfo; | ||
| 279 | bool m_presents; | ||
| 280 | std::unique_ptr<DebugRecords> m_debug; | ||
| 281 | Ref<Buffer> m_globals; | ||
| 282 | std::unique_ptr<Counters> m_counters; | ||
| 283 | std::unique_ptr<AccelSizeCache> m_accelSizes; | ||
| 284 | std::shared_ptr<ProgramCache> m_programs; | ||
| 285 | }; | ||
| 286 | |||
| 287 | } | ||
| 288 |