device/queue.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | #include "chrono_batch_render/device/queue.h" | ||
| 3 | |||
| 4 | #include <algorithm> | ||
| 5 | #include <atomic> | ||
| 6 | #include <bit> | ||
| 7 | #include <cstddef> | ||
| 8 | #include <cstring> | ||
| 9 | #include <functional> | ||
| 10 | #include <memory> | ||
| 11 | #include <mutex> | ||
| 12 | #include <optional> | ||
| 13 | #include <string> | ||
| 14 | #include <thread> | ||
| 15 | #include <utility> | ||
| 16 | #include <variant> | ||
| 17 | #include <vector> | ||
| 18 | |||
| 19 | #include <fmt/format.h> | ||
| 20 | |||
| 21 | #include "chrono_batch_render/device/device.h" | ||
| 22 | #include "chrono_batch_render/device/kernels/shared.h" | ||
| 23 | #include "chrono_batch_render/device/slang/debug.h" | ||
| 24 | #include "chrono_batch_render/util/error.h" | ||
| 25 | #include "chrono_batch_render/util/profiler.h" | ||
| 26 | #include "chrono_batch_render/util/span.h" | ||
| 27 | |||
| 28 | namespace chrono::batch_render::device { | ||
| 29 | namespace { | ||
| 30 | |||
| 31 | constexpr uint64_t kMinStagingBytes = uint64_t(64) * 1024; | ||
| 32 | constexpr uint64_t kMaxIdleStagingBytes = 256ull * 1024 * 1024; | ||
| 33 | constexpr uint64_t kMaxStagingGrowth = 64ull * 1024 * 1024; | ||
| 34 | constexpr uint64_t kStagingAlignment = 16; | ||
| 35 | constexpr uint64_t kMaxInFlight = 64; | ||
| 36 | |||
| 37 | class StagingPool { | ||
| 38 | public: | ||
| 39 | using Create = std::function<Ref<Buffer>(Memory memory, uint64_t bytes)>; | ||
| 40 | |||
| 41 | 20 | explicit StagingPool(Create create) : m_create(std::move(create)) {} | |
| 42 | |||
| 43 | 26959 | BufferRange Take(Memory memory, uint64_t bytes, bool pinned = false) { | |
| 44 | 26959 | Arena& arena = memory == Memory::Upload ? m_uploads : m_readbacks; | |
| 45 | 26959 | const uint64_t need = RoundUp(bytes, kStagingAlignment); | |
| 46 | 26959 | if (arena.buffer == nullptr || arena.cursor + need > arena.buffer->Bytes()) { | |
| 47 | 136 | const uint64_t grown = arena.buffer == nullptr | |
| 48 | 136 | ? 0 | |
| 49 | 124 | : std::min(2 * arena.buffer->Bytes(), kMaxStagingGrowth); | |
| 50 | 229 | arena = {Claim(memory, std::max(need, grown)).buffer.Get(), 0}; | |
| 51 | } | ||
| 52 | 26959 | Entry& entry = Find(arena.buffer); | |
| 53 | 26959 | if (pinned) { | |
| 54 | 1041 | entry.pins.push_back(std::this_thread::get_id()); | |
| 55 | } else { | ||
| 56 | 25918 | entry.busyUntil = kQueued; | |
| 57 | } | ||
| 58 | |||
| 59 | 26959 | const uint64_t offset = arena.cursor; | |
| 60 | 26959 | arena.cursor += need; | |
| 61 | 26959 | return arena.buffer->Slice(offset, bytes); | |
| 62 | } | ||
| 63 | |||
| 64 | 80099 | void Submitted(uint64_t value) { | |
| 65 | 977703 | for (Entry& entry : m_entries) { | |
| 66 | 897604 | if (entry.busyUntil == kQueued) { | |
| 67 | 25491 | entry.busyUntil = value; | |
| 68 | } | ||
| 69 | } | ||
| 70 | 80099 | } | |
| 71 | |||
| 72 | 63639 | void Resolve(std::thread::id thread, uint64_t value) { | |
| 73 | 770080 | for (Entry& entry : m_entries) { | |
| 74 | 706441 | const auto pinned = std::ranges::remove(entry.pins, thread); | |
| 75 | 706441 | if (pinned.begin() != entry.pins.end()) { | |
| 76 | 610 | entry.pins.erase(pinned.begin(), pinned.end()); | |
| 77 | 610 | if (entry.busyUntil != kQueued) { | |
| 78 | 1189 | entry.busyUntil = std::max(entry.busyUntil, value); | |
| 79 | } | ||
| 80 | } | ||
| 81 | } | ||
| 82 | 63639 | } | |
| 83 | |||
| 84 | 111868 | std::vector<Ref<Buffer>> Trim(uint64_t completed) { | |
| 85 | 111868 | m_completed = std::max(m_completed, completed); | |
| 86 | 111868 | std::vector<Ref<Buffer>> dropped; | |
| 87 | 111868 | uint64_t idle = 0; | |
| 88 | 1397666 | for (const Entry& entry : m_entries) { | |
| 89 | 1285798 | idle += Idle(entry) ? entry.buffer->Bytes() : 0; | |
| 90 | } | ||
| 91 | |||
| 92 | 111870 | while (idle > kMaxIdleStagingBytes) { | |
| 93 | 2 | auto largest = m_entries.end(); | |
| 94 | 52 | for (auto it = m_entries.begin(); it != m_entries.end(); ++it) { | |
| 95 | 50 | if (Idle(*it) && (largest == m_entries.end() || | |
| 96 | 44 | it->buffer->Bytes() > largest->buffer->Bytes())) { | |
| 97 | 22 | largest = it; | |
| 98 | } | ||
| 99 | } | ||
| 100 | 2 | idle -= largest->buffer->Bytes(); | |
| 101 | 2 | dropped.push_back(std::move(largest->buffer)); | |
| 102 | 2 | m_entries.erase(largest); | |
| 103 | } | ||
| 104 | 111868 | return dropped; | |
| 105 | ✗ | } | |
| 106 | |||
| 107 | 38 | std::vector<Ref<Buffer>> Clear() { | |
| 108 | 38 | std::vector<Ref<Buffer>> all; | |
| 109 | 38 | all.reserve(m_entries.size()); | |
| 110 | 159 | for (Entry& entry : m_entries) { | |
| 111 | 121 | all.push_back(std::move(entry.buffer)); | |
| 112 | } | ||
| 113 | 38 | m_entries.clear(); | |
| 114 | 38 | m_uploads = {}; | |
| 115 | 38 | m_readbacks = {}; | |
| 116 | 38 | return all; | |
| 117 | ✗ | } | |
| 118 | |||
| 119 | private: | ||
| 120 | static constexpr uint64_t kQueued = ~uint64_t(0); | ||
| 121 | |||
| 122 | 265 | struct Entry { | |
| 123 | Ref<Buffer> buffer; | ||
| 124 | uint64_t busyUntil = 0; | ||
| 125 | std::vector<std::thread::id> pins; | ||
| 126 | }; | ||
| 127 | |||
| 128 | 20 | struct Arena { | |
| 129 | Buffer* buffer = nullptr; | ||
| 130 | uint64_t cursor = 0; | ||
| 131 | }; | ||
| 132 | |||
| 133 | 1285890 | bool Idle(const Entry& entry) const { | |
| 134 | 1285890 | return entry.buffer->RefCount() == 1 && entry.pins.empty() && | |
| 135 | 2517540 | entry.busyUntil <= m_completed && entry.buffer.Get() != m_uploads.buffer && | |
| 136 | 1120818 | entry.buffer.Get() != m_readbacks.buffer; | |
| 137 | } | ||
| 138 | |||
| 139 | 26959 | Entry& Find(const Buffer* buffer) { | |
| 140 | 26959 | return *std::find_if(m_entries.begin(), m_entries.end(), | |
| 141 | 100552 | [buffer](const Entry& entry) { return entry.buffer.Get() == buffer; }); | |
| 142 | } | ||
| 143 | |||
| 144 | 136 | Entry& Claim(Memory memory, uint64_t need) { | |
| 145 | 136 | Entry* best = nullptr; | |
| 146 | 1192 | for (Entry& entry : m_entries) { | |
| 147 | 1098 | if (entry.buffer->GetMemory() == memory && entry.buffer->Bytes() >= need && | |
| 148 | 1113 | Idle(entry) && (best == nullptr || entry.buffer->Bytes() < best->buffer->Bytes())) { | |
| 149 | best = &entry; | ||
| 150 | } | ||
| 151 | } | ||
| 152 | 136 | if (best != nullptr) { | |
| 153 | return *best; | ||
| 154 | } | ||
| 155 | 246 | m_entries.push_back({m_create(memory, std::max(kMinStagingBytes, std::bit_ceil(need)))}); | |
| 156 | 123 | return m_entries.back(); | |
| 157 | 246 | } | |
| 158 | |||
| 159 | Create m_create; | ||
| 160 | std::vector<Entry> m_entries; | ||
| 161 | Arena m_uploads; | ||
| 162 | Arena m_readbacks; | ||
| 163 | uint64_t m_completed = 0; | ||
| 164 | }; | ||
| 165 | |||
| 166 | 2790 | BufferRange StageUpload(StagingPool& staging, const void* data, uint64_t bytes, | |
| 167 | bool pinned = false) { | ||
| 168 | 2790 | const BufferRange range = staging.Take(Memory::Upload, bytes, pinned); | |
| 169 | 2790 | std::memcpy(static_cast<std::byte*>(range.GetBuffer().Mapped()) + range.Offset(), data, bytes); | |
| 170 | 2790 | return range; | |
| 171 | } | ||
| 172 | |||
| 173 | 111886 | void Free(std::vector<std::shared_ptr<void>>& freed) { | |
| 174 | 196710 | for (std::shared_ptr<void>& retired : freed) { | |
| 175 | 84824 | retired.reset(); | |
| 176 | } | ||
| 177 | 111886 | } | |
| 178 | |||
| 179 | ✗ | std::string Describe(const Command& command) { | |
| 180 | ✗ | return std::visit( | |
| 181 | ✗ | Overloaded{ | |
| 182 | ✗ | [](const CopyCommand& c) { | |
| 183 | ✗ | return fmt::format("a copy of {} bytes from buffer \"{}\" to buffer \"{}\"", | |
| 184 | ✗ | c.bytes, c.src->Label(), c.dst->Label()); | |
| 185 | }, | ||
| 186 | ✗ | [](const TextureCopyCommand& c) { | |
| 187 | ✗ | return fmt::format("a copy from buffer \"{}\" to level {} of texture \"{}\"", | |
| 188 | ✗ | c.src->Label(), c.level, c.dst->Label()); | |
| 189 | }, | ||
| 190 | ✗ | [](const FillCommand& c) { | |
| 191 | ✗ | return fmt::format("a fill of buffer \"{}\"", c.dst->Label()); | |
| 192 | }, | ||
| 193 | ✗ | [](const UpdateCommand& c) { | |
| 194 | ✗ | return fmt::format("an update of buffer \"{}\"", c.dst->Label()); | |
| 195 | }, | ||
| 196 | ✗ | [](const DispatchCommand& c) { | |
| 197 | ✗ | return fmt::format("{}kernel {} of program \"{}\"", | |
| 198 | ✗ | c.counts != nullptr ? "an indirect dispatch of " : "", | |
| 199 | ✗ | c.kernel->Name(), c.kernel->GetProgram().Label()); | |
| 200 | }, | ||
| 201 | ✗ | [](const BuildAccelCommand& c) { | |
| 202 | ✗ | return fmt::format("a build of acceleration structure \"{}\"", c.dst->Label()); | |
| 203 | }, | ||
| 204 | ✗ | [](const BarrierCommand&) { return std::string("a barrier"); }, | |
| 205 | ✗ | [](const ZoneCommand&) { return std::string("a zone"); }, | |
| 206 | }, | ||
| 207 | ✗ | command); | |
| 208 | } | ||
| 209 | |||
| 210 | 11 | std::vector<uint32_t> Depths(const CommandBuffer& commands) { | |
| 211 | 11 | std::vector<uint32_t> depths(commands.Zones().size()); | |
| 212 | 11 | uint32_t open = 0; | |
| 213 | 673 | for (const Command& command : commands.Commands()) { | |
| 214 | 1324 | if (const auto* zone = std::get_if<ZoneCommand>(&command)) { | |
| 215 | 36 | if (zone->begin) { | |
| 216 | 18 | depths[zone->index] = open++; | |
| 217 | } else { | ||
| 218 | 18 | --open; | |
| 219 | } | ||
| 220 | } | ||
| 221 | } | ||
| 222 | 11 | return depths; | |
| 223 | } | ||
| 224 | |||
| 225 | } | ||
| 226 | |||
| 227 | struct Queue::State { | ||
| 228 | 20 | State(Queue& queue, StagingPool::Create create) | |
| 229 | 20 | : queued(queue, true), staging(std::move(create)) {} | |
| 230 | |||
| 231 | std::mutex submitMutex; | ||
| 232 | std::atomic<std::thread::id> exclusive{}; | ||
| 233 | std::atomic<uint64_t> submitted{0}; | ||
| 234 | std::atomic<int64_t> openRecordings{0}; | ||
| 235 | std::atomic<int64_t> batches{0}; | ||
| 236 | std::atomic<bool> lost{false}; | ||
| 237 | |||
| 238 | std::mutex mutex; | ||
| 239 | uint64_t taken = 0; | ||
| 240 | uint64_t completed = 0; | ||
| 241 | CommandEncoder queued; | ||
| 242 | StagingPool staging; | ||
| 243 | std::vector<std::pair<uint64_t, std::shared_ptr<void>>> retired; | ||
| 244 | bool closing = false; | ||
| 245 | |||
| 246 | struct Timed { | ||
| 247 | uint64_t value; | ||
| 248 | uint64_t frame; | ||
| 249 | uint64_t submittedNs; | ||
| 250 | std::vector<ZoneId> zones; | ||
| 251 | std::vector<uint32_t> depths; | ||
| 252 | std::shared_ptr<const Timestamps> timestamps; | ||
| 253 | }; | ||
| 254 | |||
| 255 | mutable std::mutex zoneMutex; | ||
| 256 | std::vector<Timed> timed; | ||
| 257 | std::optional<uint32_t> track; | ||
| 258 | }; | ||
| 259 | |||
| 260 | 127290 | void Queue::OpenRecording() { | |
| 261 | 127290 | ++m_state->openRecordings; | |
| 262 | 127290 | } | |
| 263 | |||
| 264 | 127290 | void Queue::CloseRecording() { | |
| 265 | 127290 | --m_state->openRecordings; | |
| 266 | 127290 | } | |
| 267 | |||
| 268 | 25954 | bool Operation::Ready() const { | |
| 269 | 25954 | if (queue == nullptr || value == 0) { | |
| 270 | return true; | ||
| 271 | } | ||
| 272 | 25954 | queue->Reach(value); | |
| 273 | 25954 | return queue->Completed() >= value; | |
| 274 | } | ||
| 275 | |||
| 276 | 29792 | void Operation::Wait() const { | |
| 277 | 29792 | if (queue != nullptr) { | |
| 278 | 29633 | queue->Wait(value); | |
| 279 | } | ||
| 280 | 29792 | } | |
| 281 | |||
| 282 | 4506 | void Operation::Wait(Stream stream) const { | |
| 283 | 4506 | if (stream.IsNone() || queue == nullptr || value == 0) { | |
| 284 | return; | ||
| 285 | } | ||
| 286 | 700 | if (stream.IsHost()) { | |
| 287 | 700 | Wait(); | |
| 288 | 700 | return; | |
| 289 | } | ||
| 290 | ✗ | if (!queue->SharesStreams()) { | |
| 291 | 4506 | if (stream.IsDefault()) { | |
| 292 | return; | ||
| 293 | } | ||
| 294 | ✗ | throw UnsupportedError("{} shares no streams with CUDA or HIP, so none waits for it", | |
| 295 | ✗ | queue->GetDevice().Adapter().alias); | |
| 296 | } | ||
| 297 | ✗ | queue->Reach(value); | |
| 298 | ✗ | queue->StreamWait(stream.Handle(), value); | |
| 299 | } | ||
| 300 | |||
| 301 | 24422 | Download::Download() = default; | |
| 302 | 49061 | Download::~Download() = default; | |
| 303 | 8 | Download::Download(const Download& other) = default; | |
| 304 | 454 | Download::Download(Download&& other) noexcept = default; | |
| 305 | ✗ | Download& Download::operator=(const Download& other) = default; | |
| 306 | 250 | Download& Download::operator=(Download&& other) noexcept = default; | |
| 307 | |||
| 308 | 25109 | Span<const std::byte> Download::Bytes() const { | |
| 309 | 25109 | if (!m_staging) { | |
| 310 | 2 | return {}; | |
| 311 | } | ||
| 312 | 25107 | if (!Ready()) { | |
| 313 | ✗ | throw Error("a download's bytes aren't here yet; wait for it, or check Ready(), first"); | |
| 314 | } | ||
| 315 | 25107 | return {static_cast<const std::byte*>(m_staging->Mapped()) + m_offset, m_bytes}; | |
| 316 | } | ||
| 317 | |||
| 318 | 20 | Queue::Queue(Device& device, bool runsKernels) | |
| 319 | 20 | : m_device(device), m_runsKernels(runsKernels), | |
| 320 | 163 | m_state(std::make_unique<State>(*this, [&device](Memory memory, uint64_t bytes) { | |
| 321 | 123 | Ref<Buffer> buffer = | |
| 322 | 123 | device.CreateBuffer({.bytes = bytes, .memory = memory, .label = "staging"}); | |
| 323 | 123 | Device::Adopt(*buffer); | |
| 324 | 123 | return buffer; | |
| 325 | 266 | })) {} | |
| 326 | |||
| 327 | 20 | Queue::~Queue() = default; | |
| 328 | |||
| 329 | 63646 | CommandEncoder Queue::CreateEncoder() { | |
| 330 | 63646 | return CommandEncoder(*this, false); | |
| 331 | } | ||
| 332 | |||
| 333 | 63635 | Operation Queue::Submit(CommandBuffer&& commands) { | |
| 334 | 63635 | return SubmitCommands(commands); | |
| 335 | } | ||
| 336 | |||
| 337 | 4 | Operation Queue::Submit(const CommandBuffer& commands) { | |
| 338 | 4 | if (!commands.m_reusable) { | |
| 339 | ✗ | throw Error("a command buffer that isn't reusable was submitted to run again; finish it " | |
| 340 | ✗ | "with {{.reusable = true}}, or submit it once with std::move"); | |
| 341 | } | ||
| 342 | 4 | return SubmitCommands(commands); | |
| 343 | } | ||
| 344 | |||
| 345 | 7716 | Operation Queue::Mark() { | |
| 346 | 15432 | return Submit(CreateEncoder().Finish()); | |
| 347 | } | ||
| 348 | |||
| 349 | 63639 | Operation Queue::SubmitCommands(const CommandBuffer& commands) { | |
| 350 | 63639 | if (commands.m_queue != this) { | |
| 351 | ✗ | throw Error("a command buffer recorded for another queue was submitted"); | |
| 352 | } | ||
| 353 | 63639 | if (commands.m_reusable) { | |
| 354 | 4 | if (commands.m_submission > Finished()) { | |
| 355 | 1 | Wait(commands.m_submission); | |
| 356 | } | ||
| 357 | 4 | Collect(Finished()); | |
| 358 | } | ||
| 359 | |||
| 360 | 63639 | if (m_state->batches > 0 && !m_device.m_synchronous && commands.m_reusable) { | |
| 361 | 4 | Run(CommandBuffer(*this, true)); | |
| 362 | 63635 | } else if (m_state->batches > 0 && !m_device.m_synchronous) { | |
| 363 | 7895 | { | |
| 364 | 7895 | const std::lock_guard lock(m_state->mutex); | |
| 365 | 7895 | m_state->queued.Append(commands); | |
| 366 | 7895 | commands.m_submission = m_state->taken + 1; | |
| 367 | 7895 | m_state->staging.Resolve(std::this_thread::get_id(), commands.m_submission); | |
| 368 | 7895 | } | |
| 369 | 7895 | m_device.CountTraffic({.inlineBytes = commands.m_data.size()}); | |
| 370 | 7895 | if (!commands.m_reusable) { | |
| 371 | 7895 | const_cast<CommandBuffer&>(commands).Close(); | |
| 372 | } | ||
| 373 | 7895 | return {this, commands.m_submission}; | |
| 374 | } | ||
| 375 | |||
| 376 | 55744 | commands.m_submission = m_device.m_synchronous ? RunEach(commands) : Run(commands); | |
| 377 | 55744 | { | |
| 378 | 55744 | const std::lock_guard lock(m_state->mutex); | |
| 379 | 55744 | m_state->staging.Resolve(std::this_thread::get_id(), commands.m_submission); | |
| 380 | 55744 | } | |
| 381 | 55744 | if (!commands.m_reusable) { | |
| 382 | 55740 | const_cast<CommandBuffer&>(commands).Close(); | |
| 383 | } | ||
| 384 | |||
| 385 | 55744 | Collect(Finished()); | |
| 386 | 55744 | Throttle(commands.m_submission); | |
| 387 | 55744 | return {this, commands.m_submission}; | |
| 388 | } | ||
| 389 | |||
| 390 | 55744 | void Queue::Throttle(uint64_t value) { | |
| 391 | 55744 | if (value <= kMaxInFlight || m_device.m_synchronous) { | |
| 392 | return; | ||
| 393 | } | ||
| 394 | 55234 | const uint64_t oldest = value - kMaxInFlight; | |
| 395 | 55234 | if (Finished() >= oldest) { | |
| 396 | return; | ||
| 397 | } | ||
| 398 | 7 | m_device.CountTraffic({.hostWaits = 1}); | |
| 399 | 7 | WaitFor(oldest); | |
| 400 | 7 | Collect(Finished()); | |
| 401 | } | ||
| 402 | |||
| 403 | 1924 | void Queue::BeginBatch() { | |
| 404 | 1924 | ++m_state->batches; | |
| 405 | 1924 | } | |
| 406 | |||
| 407 | 1924 | Operation Queue::EndBatch() { | |
| 408 | 1924 | if (--m_state->batches > 0) { | |
| 409 | 36 | const std::lock_guard lock(m_state->mutex); | |
| 410 | 36 | return {this, m_state->taken + 1}; | |
| 411 | 36 | } | |
| 412 | 1888 | return Mark(); | |
| 413 | } | ||
| 414 | |||
| 415 | 56111 | void Queue::Reach(uint64_t value) { | |
| 416 | 56111 | if (value > m_state->submitted) { | |
| 417 | 24351 | Run(CommandBuffer(*this, true)); | |
| 418 | } | ||
| 419 | 56111 | } | |
| 420 | |||
| 421 | 56190 | void Queue::Follow(Stream stream) { | |
| 422 | 56190 | if (!stream.IsCompute()) { | |
| 423 | 56190 | return; | |
| 424 | } | ||
| 425 | ✗ | if (!SharesStreams()) { | |
| 426 | 56190 | if (stream.IsDefault()) { | |
| 427 | return; | ||
| 428 | } | ||
| 429 | ✗ | throw UnsupportedError("{} shares no streams with CUDA or HIP, so it follows none", | |
| 430 | ✗ | m_device.Adapter().alias); | |
| 431 | } | ||
| 432 | ✗ | const std::unique_lock submitLock = LockSubmissions(); | |
| 433 | ✗ | FollowStream(stream.Handle()); | |
| 434 | ✗ | } | |
| 435 | |||
| 436 | 4 | void Queue::Follow(const Operation& operation) { | |
| 437 | 4 | Queue* other = operation.queue; | |
| 438 | 4 | if (other == nullptr || other == this || operation.value == 0) { | |
| 439 | return; | ||
| 440 | } | ||
| 441 | ✗ | if (&other->m_device == &m_device) { | |
| 442 | ✗ | other->Reach(operation.value); | |
| 443 | ✗ | const std::unique_lock submitLock = LockSubmissions(); | |
| 444 | ✗ | FollowQueue(*other, operation.value); | |
| 445 | ✗ | return; | |
| 446 | ✗ | } | |
| 447 | ✗ | if (other->m_device.Adapter().uuid == m_device.Adapter().uuid) { | |
| 448 | ✗ | if (void* stream = NativeStream(); stream != nullptr && other->SharesStreams()) { | |
| 449 | ✗ | operation.Wait(Stream::Compute(stream)); | |
| 450 | ✗ | return; | |
| 451 | } | ||
| 452 | ✗ | if (void* stream = other->NativeStream(); stream != nullptr && SharesStreams()) { | |
| 453 | ✗ | other->Reach(operation.value); | |
| 454 | ✗ | Follow(Stream::Compute(stream)); | |
| 455 | ✗ | return; | |
| 456 | } | ||
| 457 | ✗ | other->Reach(operation.value); | |
| 458 | ✗ | const std::unique_lock submitLock = LockSubmissions(); | |
| 459 | ✗ | if (FollowDevice(*other, operation.value)) { | |
| 460 | ✗ | return; | |
| 461 | } | ||
| 462 | ✗ | } | |
| 463 | ✗ | operation.Wait(); | |
| 464 | } | ||
| 465 | |||
| 466 | 80100 | std::unique_lock<std::mutex> Queue::LockSubmissions() { | |
| 467 | 80100 | if (m_state->exclusive.load() == std::this_thread::get_id()) { | |
| 468 | ✗ | throw Error("{} got work inside Exclusive, which holds its queue; submit it after " | |
| 469 | "Exclusive returns", | ||
| 470 | ✗ | m_device.Adapter().alias); | |
| 471 | } | ||
| 472 | 80100 | return std::unique_lock(m_state->submitMutex); | |
| 473 | } | ||
| 474 | |||
| 475 | 1 | void Queue::Exclusive(const std::function<void()>& use) { | |
| 476 | 1 | const std::unique_lock submitLock = LockSubmissions(); | |
| 477 | 1 | m_state->exclusive = std::this_thread::get_id(); | |
| 478 | 1 | try { | |
| 479 | 1 | use(); | |
| 480 | ✗ | } catch (...) { | |
| 481 | ✗ | m_state->exclusive = std::thread::id(); | |
| 482 | ✗ | throw; | |
| 483 | ✗ | } | |
| 484 | 1 | m_state->exclusive = std::thread::id(); | |
| 485 | 1 | } | |
| 486 | |||
| 487 | ✗ | Operation Queue::RunNative(const void* native, const char* zone) { | |
| 488 | ✗ | CommandEncoder timed(*this, true); | |
| 489 | ✗ | if (zone != nullptr) { | |
| 490 | ✗ | timed.BeginZone(ZoneId::Named(zone)); | |
| 491 | ✗ | timed.EndZone(); | |
| 492 | } | ||
| 493 | ✗ | const uint64_t value = Run(timed.Finish(), native); | |
| 494 | ✗ | if (m_device.m_synchronous) { | |
| 495 | ✗ | Wait(value); | |
| 496 | } | ||
| 497 | ✗ | Collect(Finished()); | |
| 498 | ✗ | Throttle(value); | |
| 499 | ✗ | return {this, value}; | |
| 500 | ✗ | } | |
| 501 | |||
| 502 | ✗ | bool Queue::FollowDevice(Queue&, uint64_t) { | |
| 503 | ✗ | return false; | |
| 504 | } | ||
| 505 | |||
| 506 | ✗ | void Queue::FollowQueue(Queue& other, uint64_t value) { | |
| 507 | ✗ | Operation{&other, value}.Wait(); | |
| 508 | ✗ | } | |
| 509 | |||
| 510 | ✗ | void Queue::ExecuteNative(const CommandBuffer&, uint64_t, const void*, const Timestamps*) { | |
| 511 | ✗ | throw UnsupportedError("{} runs no native commands", m_device.Adapter().alias); | |
| 512 | } | ||
| 513 | |||
| 514 | ✗ | void Queue::FollowStream(void*) { | |
| 515 | ✗ | throw UnsupportedError("{} shares no streams with CUDA or HIP", m_device.Adapter().alias); | |
| 516 | } | ||
| 517 | |||
| 518 | ✗ | void Queue::StreamWait(void*, uint64_t) { | |
| 519 | ✗ | throw UnsupportedError("{} shares no streams with CUDA or HIP", m_device.Adapter().alias); | |
| 520 | } | ||
| 521 | |||
| 522 | 80099 | uint64_t Queue::Run(const CommandBuffer& commands, const void* native) { | |
| 523 | 80099 | CHRONO_BATCH_RENDER_ZONE("Queue/Run"); | |
| 524 | 80099 | State& state = *m_state; | |
| 525 | 80099 | const std::unique_lock submitLock = LockSubmissions(); | |
| 526 | 80099 | if (state.lost) { | |
| 527 | ✗ | throw DeviceLostError("{} was lost; no more work runs on it", m_device.Adapter().alias); | |
| 528 | } | ||
| 529 | |||
| 530 | 80099 | uint64_t value = 0; | |
| 531 | ✗ | CommandBuffer queued = [&] { | |
| 532 | 80099 | const std::lock_guard lock(state.mutex); | |
| 533 | 80099 | value = ++state.taken; | |
| 534 | 80099 | state.staging.Submitted(value); | |
| 535 | 80099 | if (m_runsKernels) { | |
| 536 | 80099 | state.queued.Fill(m_device.Debug().Submission(), uint32_t(value)); | |
| 537 | } | ||
| 538 | 80099 | return state.queued.Finish(); | |
| 539 | 160198 | }(); | |
| 540 | 80099 | m_device.CountTraffic({.inlineBytes = commands.m_data.size(), .submissions = 1}); | |
| 541 | |||
| 542 | 80099 | if (native == nullptr && !queued.m_zones.empty()) { | |
| 543 | 2250 | CommandEncoder merged(*this, true); | |
| 544 | 2250 | merged.Append(queued); | |
| 545 | 2250 | merged.Append(commands); | |
| 546 | 2250 | queued = CommandBuffer(*this, true); | |
| 547 | 2250 | return Launch(queued, merged.Finish(), value, native); | |
| 548 | 2250 | } | |
| 549 | 77849 | return Launch(queued, commands, value, native); | |
| 550 | 160198 | } | |
| 551 | |||
| 552 | 80099 | uint64_t Queue::Launch(const CommandBuffer& queued, const CommandBuffer& commands, uint64_t value, | |
| 553 | const void* native) { | ||
| 554 | 80099 | State& state = *m_state; | |
| 555 | 80099 | try { | |
| 556 | 80099 | if (native != nullptr) { | |
| 557 | ✗ | std::shared_ptr<Timestamps> timestamps = TimestampsFor(commands); | |
| 558 | ✗ | ExecuteNative(queued, value, native, timestamps.get()); | |
| 559 | ✗ | if (timestamps) { | |
| 560 | ✗ | const std::lock_guard lock(state.zoneMutex); | |
| 561 | ✗ | state.timed.push_back({value, profiler::CurrentFrame(), profiler::Now(), | |
| 562 | ✗ | commands.m_zones, Depths(commands), std::move(timestamps)}); | |
| 563 | ✗ | } | |
| 564 | ✗ | state.submitted = value; | |
| 565 | ✗ | return value; | |
| 566 | ✗ | } | |
| 567 | 80099 | std::shared_ptr<Timestamps> timestamps = TimestampsFor(commands); | |
| 568 | 80099 | Execute(queued, commands, timestamps.get(), value); | |
| 569 | 80099 | if (timestamps) { | |
| 570 | 11 | const std::lock_guard lock(state.zoneMutex); | |
| 571 | 11 | state.timed.push_back({value, profiler::CurrentFrame(), profiler::Now(), | |
| 572 | 11 | commands.m_zones, Depths(commands), std::move(timestamps)}); | |
| 573 | 11 | } | |
| 574 | ✗ | } catch (...) { | |
| 575 | ✗ | state.lost = true; | |
| 576 | ✗ | throw; | |
| 577 | ✗ | } | |
| 578 | |||
| 579 | 80099 | state.submitted = value; | |
| 580 | 80099 | return value; | |
| 581 | 11 | } | |
| 582 | |||
| 583 | ✗ | uint64_t Queue::RunEach(const CommandBuffer& commands) { | |
| 584 | ✗ | uint64_t value = 0; | |
| 585 | ✗ | for (const Command& command : commands.m_commands) { | |
| 586 | ✗ | if (std::holds_alternative<BarrierCommand>(command) || | |
| 587 | ✗ | std::holds_alternative<ZoneCommand>(command)) { | |
| 588 | ✗ | continue; | |
| 589 | } | ||
| 590 | ✗ | const CommandBuffer one = Isolate(commands, command); | |
| 591 | ✗ | value = Run(one); | |
| 592 | ✗ | try { | |
| 593 | ✗ | Wait(value); | |
| 594 | ✗ | } catch (const DeviceLostError& lost) { | |
| 595 | ✗ | throw DeviceLostError("{}, running {}", lost.what(), Describe(command)); | |
| 596 | ✗ | } | |
| 597 | ✗ | } | |
| 598 | ✗ | return value != 0 ? value : Run(CommandBuffer(*this, true)); | |
| 599 | } | ||
| 600 | |||
| 601 | ✗ | CommandBuffer Queue::Isolate(const CommandBuffer& commands, const Command& command) { | |
| 602 | ✗ | CommandBuffer one(*this, true); | |
| 603 | ✗ | Command& alone = one.m_commands.emplace_back(command); | |
| 604 | ✗ | const auto move = [&](size_t& data, size_t bytes) { | |
| 605 | ✗ | one.m_data.assign(commands.m_data.begin() + ptrdiff_t(data), | |
| 606 | ✗ | commands.m_data.begin() + ptrdiff_t(data + RoundUp(bytes, 4))); | |
| 607 | ✗ | data = 0; | |
| 608 | ✗ | }; | |
| 609 | ✗ | std::visit(Overloaded{ | |
| 610 | ✗ | [&](UpdateCommand& c) { move(c.data, c.bytes); }, | |
| 611 | ✗ | [&](DispatchCommand& c) { move(c.data, c.paramBytes); }, | |
| 612 | [](auto&) {}, | ||
| 613 | }, | ||
| 614 | alone); | ||
| 615 | ✗ | return one; | |
| 616 | ✗ | } | |
| 617 | |||
| 618 | 80099 | std::shared_ptr<Queue::Timestamps> Queue::TimestampsFor(const CommandBuffer& commands) { | |
| 619 | 80099 | if (commands.m_zones.empty() || !m_device.m_profiling || !profiler::Enabled()) { | |
| 620 | 80088 | return nullptr; | |
| 621 | } | ||
| 622 | |||
| 623 | 11 | const auto count = uint32_t(2 * commands.m_zones.size()); | |
| 624 | 11 | if (!commands.m_reusable) { | |
| 625 | 11 | return CreateTimestamps(count); | |
| 626 | } | ||
| 627 | ✗ | std::shared_ptr<void>& kept = commands.m_recorded.timestamps; | |
| 628 | ✗ | if (!kept) { | |
| 629 | ✗ | kept = CreateTimestamps(count); | |
| 630 | } | ||
| 631 | ✗ | return std::static_pointer_cast<Timestamps>(kept); | |
| 632 | } | ||
| 633 | |||
| 634 | 111886 | void Queue::CollectZones(uint64_t finished) const { | |
| 635 | 111886 | State& state = *m_state; | |
| 636 | 111886 | std::vector<State::Timed> read; | |
| 637 | 111886 | { | |
| 638 | 111886 | const std::lock_guard lock(state.zoneMutex); | |
| 639 | 111886 | const auto done = std::ranges::find_if( | |
| 640 | 111907 | state.timed, [finished](const State::Timed& run) { return run.value > finished; }); | |
| 641 | 111886 | if (done == state.timed.begin()) { | |
| 642 | 111876 | return; | |
| 643 | } | ||
| 644 | 10 | if (!state.track) { | |
| 645 | 1 | state.track = profiler::AddGpuTrack(m_device.Adapter().alias); | |
| 646 | } | ||
| 647 | 10 | const uint32_t track = *state.track; | |
| 648 | |||
| 649 | 10 | std::vector<uint64_t> ns; | |
| 650 | 21 | for (auto run = state.timed.begin(); run != done; ++run) { | |
| 651 | 11 | ns.resize(2 * run->zones.size()); | |
| 652 | 11 | if (!run->timestamps->Read(ns)) { | |
| 653 | ✗ | const uint64_t origin = ns[0]; | |
| 654 | ✗ | for (uint64_t& time : ns) { | |
| 655 | ✗ | time = time - origin + run->submittedNs; | |
| 656 | } | ||
| 657 | } | ||
| 658 | 29 | for (size_t i = 0; i < run->zones.size(); ++i) { | |
| 659 | 18 | profiler::AddZone(run->frame, | |
| 660 | 18 | {run->zones[i], track, run->depths[i], ns[2 * i], ns[2 * i + 1]}); | |
| 661 | } | ||
| 662 | } | ||
| 663 | |||
| 664 | 10 | read.assign(std::make_move_iterator(state.timed.begin()), std::make_move_iterator(done)); | |
| 665 | 10 | state.timed.erase(state.timed.begin(), done); | |
| 666 | 10 | } | |
| 667 | 111886 | } | |
| 668 | |||
| 669 | 207488 | void Queue::Write(BufferRange dst, const void* data) { | |
| 670 | 207488 | m_device.Require(dst.GetBuffer(), "Write"); | |
| 671 | 207488 | State& state = *m_state; | |
| 672 | 207488 | const std::lock_guard lock(state.mutex); | |
| 673 | 207488 | constexpr uint64_t kInlineWriteBytes = 4096; | |
| 674 | 207488 | if (dst.Bytes() <= kInlineWriteBytes && dst.Offset() % 4 == 0 && dst.Bytes() % 4 == 0) { | |
| 675 | 205740 | state.queued.Update(dst, data); | |
| 676 | 1748 | } else if (dst.Bytes() > 0) { | |
| 677 | 1748 | state.queued.Copy(StageUpload(state.staging, data, dst.Bytes()), dst); | |
| 678 | } | ||
| 679 | 207488 | m_device.CountTraffic({.uploadBytes = dst.Bytes()}); | |
| 680 | 207488 | } | |
| 681 | |||
| 682 | 1041 | BufferRange Queue::Upload(const void* data, uint64_t bytes) { | |
| 683 | 1041 | State& state = *m_state; | |
| 684 | 1041 | const std::lock_guard lock(state.mutex); | |
| 685 | 1041 | m_device.CountTraffic({.uploadBytes = bytes}); | |
| 686 | 1041 | return StageUpload(state.staging, data, bytes, true); | |
| 687 | 1041 | } | |
| 688 | |||
| 689 | 1 | void Queue::Write(Texture& dst, uint32_t level, const void* data) { | |
| 690 | 1 | m_device.Require(dst, "Write"); | |
| 691 | 1 | const uint64_t bytes = dst.LevelBytes(level); | |
| 692 | 1 | State& state = *m_state; | |
| 693 | 1 | const std::lock_guard lock(state.mutex); | |
| 694 | 1 | state.queued.Copy(StageUpload(state.staging, data, bytes), dst, level); | |
| 695 | 1 | m_device.CountTraffic({.uploadBytes = bytes}); | |
| 696 | 1 | } | |
| 697 | |||
| 698 | 24171 | Download Queue::ReadAsync(BufferRange src) { | |
| 699 | 24171 | m_device.Require(src.GetBuffer(), "ReadAsync"); | |
| 700 | 24171 | Download download; | |
| 701 | 24171 | if (src.Bytes() == 0) { | |
| 702 | return download; | ||
| 703 | } | ||
| 704 | |||
| 705 | 24169 | State& state = *m_state; | |
| 706 | 24169 | const std::lock_guard lock(state.mutex); | |
| 707 | 24169 | const BufferRange staging = state.staging.Take(Memory::Readback, src.Bytes()); | |
| 708 | 24169 | state.queued.Copy(src, staging); | |
| 709 | 24169 | m_device.CountTraffic({.readbackBytes = src.Bytes()}); | |
| 710 | |||
| 711 | 24169 | static_cast<Operation&>(download) = {this, state.taken + 1}; | |
| 712 | 24169 | download.m_device = Ref<Device>(&m_device); | |
| 713 | 24169 | download.m_staging = Ref<Buffer>(&staging.GetBuffer()); | |
| 714 | 24169 | download.m_offset = staging.Offset(); | |
| 715 | 24169 | download.m_bytes = src.Bytes(); | |
| 716 | 24169 | return download; | |
| 717 | 24169 | } | |
| 718 | |||
| 719 | 5043 | void Queue::Read(BufferRange src, void* data) { | |
| 720 | 5043 | const Download download = ReadAsync(src); | |
| 721 | 5043 | download.Wait(); | |
| 722 | 5043 | const std::span<const std::byte> bytes = download.Bytes(); | |
| 723 | 5043 | std::memcpy(data, bytes.data(), bytes.size()); | |
| 724 | 5043 | } | |
| 725 | |||
| 726 | 27 | uint64_t Queue::Submitted() const { | |
| 727 | 27 | return m_state->submitted; | |
| 728 | } | ||
| 729 | |||
| 730 | 25956 | uint64_t Queue::Completed() const { | |
| 731 | 25956 | const uint64_t finished = Finished(); | |
| 732 | 25956 | Collect(finished); | |
| 733 | 25956 | return finished; | |
| 734 | } | ||
| 735 | |||
| 736 | 526 | void Queue::WaitIdle() { | |
| 737 | 526 | uint64_t value = 0; | |
| 738 | 526 | { | |
| 739 | 526 | const std::lock_guard lock(m_state->mutex); | |
| 740 | 526 | value = m_state->queued.m_recording.Empty() ? m_state->taken : m_state->taken + 1; | |
| 741 | 526 | } | |
| 742 | 526 | Wait(value); | |
| 743 | 526 | } | |
| 744 | |||
| 745 | 30160 | void Queue::Wait(uint64_t value) { | |
| 746 | 30160 | if (value == 0) { | |
| 747 | return; | ||
| 748 | } | ||
| 749 | 30157 | Reach(value); | |
| 750 | 30157 | if (Finished() < value) { | |
| 751 | 23193 | m_device.CountTraffic({.hostWaits = 1}); | |
| 752 | 23193 | WaitFor(value); | |
| 753 | } | ||
| 754 | |||
| 755 | 30157 | Collect(Finished()); | |
| 756 | 30157 | if (!m_runsKernels) { | |
| 757 | return; | ||
| 758 | } | ||
| 759 | 30157 | if (std::optional<std::string> failed = m_device.Debug().TakeFailures(value)) { | |
| 760 | ✗ | throw Error(*failed); | |
| 761 | 30157 | } | |
| 762 | } | ||
| 763 | |||
| 764 | 111868 | void Queue::Collect(uint64_t finished) const { | |
| 765 | 111868 | State& state = *m_state; | |
| 766 | 111868 | std::vector<std::shared_ptr<void>> freed; | |
| 767 | 111868 | std::vector<Ref<Buffer>> dropped; | |
| 768 | 111868 | { | |
| 769 | 111868 | const std::lock_guard lock(state.mutex); | |
| 770 | 111868 | state.completed = std::max(state.completed, finished); | |
| 771 | 111868 | const auto done = std::stable_partition( | |
| 772 | state.retired.begin(), state.retired.end(), | ||
| 773 | 135674 | [&](const auto& retired) { return retired.first > state.completed; }); | |
| 774 | 196373 | for (auto it = done; it != state.retired.end(); ++it) { | |
| 775 | 84505 | freed.push_back(std::move(it->second)); | |
| 776 | } | ||
| 777 | 111868 | state.retired.erase(done, state.retired.end()); | |
| 778 | |||
| 779 | 111868 | if (m_runsKernels) { | |
| 780 | 111868 | m_device.Debug().Read(state.completed, state.completed >= state.taken); | |
| 781 | } | ||
| 782 | 223736 | dropped = state.staging.Trim(state.completed); | |
| 783 | ✗ | } | |
| 784 | |||
| 785 | 111868 | CollectZones(finished); | |
| 786 | 111868 | Free(freed); | |
| 787 | 111868 | } | |
| 788 | |||
| 789 | 120320 | void Queue::Retire(Resource& resource) { | |
| 790 | 120320 | State& state = *m_state; | |
| 791 | 120320 | uint64_t value = 0; | |
| 792 | 120320 | { | |
| 793 | 120320 | const std::lock_guard lock(state.mutex); | |
| 794 | 120320 | const bool pending = state.openRecordings > 0 || !state.queued.m_recording.Empty(); | |
| 795 | 120320 | value = pending ? state.taken + 1 : state.taken; | |
| 796 | 120320 | } | |
| 797 | |||
| 798 | 120320 | Retire( | |
| 799 | 240640 | std::shared_ptr<void>(&resource, [](void* freed) { delete static_cast<Resource*>(freed); }), | |
| 800 | value); | ||
| 801 | 120320 | } | |
| 802 | |||
| 803 | 2784 | void Queue::Retire(std::shared_ptr<void> retired, const Operation& until) { | |
| 804 | 2784 | Retire(std::move(retired), until.value); | |
| 805 | 2784 | } | |
| 806 | |||
| 807 | 123106 | void Queue::Retire(std::shared_ptr<void> retired, uint64_t value) { | |
| 808 | 123106 | State& state = *m_state; | |
| 809 | 123106 | { | |
| 810 | 123106 | const std::lock_guard lock(state.mutex); | |
| 811 | 123106 | if (!state.closing && value > state.completed) { | |
| 812 | 84824 | state.retired.emplace_back(value, std::move(retired)); | |
| 813 | 84824 | return; | |
| 814 | } | ||
| 815 | 84824 | } | |
| 816 | 38282 | retired.reset(); | |
| 817 | } | ||
| 818 | |||
| 819 | 20 | void Queue::Close() noexcept { | |
| 820 | 20 | State& state = *m_state; | |
| 821 | 20 | { | |
| 822 | 20 | const std::lock_guard lock(state.mutex); | |
| 823 | 20 | state.closing = true; | |
| 824 | 20 | } | |
| 825 | |||
| 826 | 20 | try { | |
| 827 | 20 | if (state.submitted > 0) { | |
| 828 | 18 | WaitFor(state.submitted); | |
| 829 | 18 | CollectZones(state.submitted); | |
| 830 | } | ||
| 831 | ✗ | } catch (...) { // NOLINT(bugprone-empty-catch) | |
| 832 | ✗ | } | |
| 833 | |||
| 834 | 20 | { | |
| 835 | 20 | std::vector<State::Timed> timed; | |
| 836 | 20 | const std::lock_guard lock(state.zoneMutex); | |
| 837 | 20 | timed.swap(state.timed); | |
| 838 | 20 | } | |
| 839 | |||
| 840 | 18 | for (;;) { | |
| 841 | 38 | std::vector<std::shared_ptr<void>> freed; | |
| 842 | 38 | std::vector<Ref<Buffer>> staging; | |
| 843 | 38 | { | |
| 844 | 38 | const std::lock_guard lock(state.mutex); | |
| 845 | 38 | state.queued.Finish(); | |
| 846 | 38 | staging = state.staging.Clear(); | |
| 847 | 357 | for (auto& [value, retired] : state.retired) { | |
| 848 | 319 | freed.push_back(std::move(retired)); | |
| 849 | } | ||
| 850 | 38 | state.retired.clear(); | |
| 851 | 38 | } | |
| 852 | 38 | if (freed.empty() && staging.empty()) { | |
| 853 | 20 | return; | |
| 854 | } | ||
| 855 | 18 | Free(freed); | |
| 856 | 18 | staging.clear(); | |
| 857 | 38 | } | |
| 858 | } | ||
| 859 | |||
| 860 | } | ||
| 861 |