python/src/dlpack.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | #include <Python.h> | ||
| 3 | #include <fmt/format.h> | ||
| 4 | |||
| 5 | #include <algorithm> | ||
| 6 | #include <cstddef> | ||
| 7 | #include <cstdint> | ||
| 8 | #include <memory> | ||
| 9 | #include <optional> | ||
| 10 | #include <vector> | ||
| 11 | |||
| 12 | #include "bindings.h" | ||
| 13 | #include "chrono_batch_render/device/commands.h" | ||
| 14 | #include "chrono_batch_render/device/queue.h" | ||
| 15 | |||
| 16 | namespace nb = nanobind; | ||
| 17 | using namespace nb::literals; | ||
| 18 | |||
| 19 | namespace chrono::batch_render::python { | ||
| 20 | |||
| 21 | namespace { | ||
| 22 | |||
| 23 | constexpr const char* kVersioned = "dltensor_versioned"; | ||
| 24 | constexpr const char* kUnversioned = "dltensor"; | ||
| 25 | |||
| 26 | ✗ | struct Export { | |
| 27 | DLManagedTensorVersioned managed{}; | ||
| 28 | device::Tensor tensor; | ||
| 29 | std::vector<std::byte> host; | ||
| 30 | }; | ||
| 31 | |||
| 32 | 1170 | struct Producer { | |
| 33 | DLManagedTensorVersioned wrapper{}; | ||
| 34 | DLManagedTensorVersioned* versioned = nullptr; | ||
| 35 | DLManagedTensor* unversioned = nullptr; | ||
| 36 | }; | ||
| 37 | |||
| 38 | 1170 | void ReleaseProducer(DLManagedTensorVersioned* self) { | |
| 39 | 1170 | auto* producer = static_cast<Producer*>(self->manager_ctx); | |
| 40 | 1170 | if (Py_IsInitialized()) { | |
| 41 | 1170 | nb::gil_scoped_acquire gil; | |
| 42 | 1170 | if (producer->versioned != nullptr && producer->versioned->deleter != nullptr) { | |
| 43 | 1170 | producer->versioned->deleter(producer->versioned); | |
| 44 | } | ||
| 45 | 1170 | if (producer->unversioned != nullptr && producer->unversioned->deleter != nullptr) { | |
| 46 | ✗ | producer->unversioned->deleter(producer->unversioned); | |
| 47 | } | ||
| 48 | 1170 | } | |
| 49 | 1170 | delete producer; | |
| 50 | 1170 | } | |
| 51 | |||
| 52 | ✗ | DLManagedTensorVersioned* CreateExport(const device::Tensor& tensor, bool toHost) { | |
| 53 | ✗ | if (!toHost) { | |
| 54 | ✗ | return tensor.Export(); | |
| 55 | } | ||
| 56 | ✗ | auto held = std::make_unique<Export>(); | |
| 57 | ✗ | held->tensor = tensor; | |
| 58 | ✗ | held->host = tensor.ToHostBytes(); | |
| 59 | ✗ | DLManagedTensorVersioned& managed = held->managed; | |
| 60 | ✗ | DLTensor& exported = managed.dl_tensor; | |
| 61 | ✗ | exported.data = held->host.data(); | |
| 62 | ✗ | exported.device = {kDLCPU, 0}; | |
| 63 | ✗ | exported.ndim = int32_t(tensor.Shape().size()); | |
| 64 | ✗ | exported.dtype = DataType(tensor.Type()); | |
| 65 | ✗ | exported.shape = const_cast<int64_t*>(held->tensor.Shape().data()); | |
| 66 | ✗ | managed.flags = DLPACK_FLAG_BITMASK_IS_COPIED; | |
| 67 | ✗ | managed.version = {DLPACK_MAJOR_VERSION, DLPACK_MINOR_VERSION}; | |
| 68 | ✗ | managed.manager_ctx = held.get(); | |
| 69 | ✗ | managed.deleter = [](DLManagedTensorVersioned* self) { | |
| 70 | ✗ | delete static_cast<Export*>(self->manager_ctx); | |
| 71 | ✗ | }; | |
| 72 | ✗ | return &held.release()->managed; | |
| 73 | ✗ | } | |
| 74 | |||
| 75 | ✗ | device::Tensor CopyOnDevice(const device::Tensor& tensor) { | |
| 76 | ✗ | const Span<const int64_t> shape = tensor.Shape(); | |
| 77 | ✗ | const Span<const int64_t> strides = tensor.Strides(); | |
| 78 | ✗ | int64_t last = 0; | |
| 79 | ✗ | int64_t packed = 1; | |
| 80 | ✗ | for (size_t axis = shape.size(); axis-- > 0;) { | |
| 81 | ✗ | last += (shape[axis] - 1) * (strides.empty() ? packed : strides[axis]); | |
| 82 | ✗ | packed *= shape[axis]; | |
| 83 | } | ||
| 84 | ✗ | const uint64_t bytes = | |
| 85 | ✗ | tensor.Elements() == 0 ? 0 : uint64_t(last + 1) * device::ElementBytes(tensor.Type()); | |
| 86 | ✗ | device::Device& device = tensor.GetBuffer().GetDevice(); | |
| 87 | ✗ | const Ref<device::Buffer> copied = device.CreateBuffer({.bytes = std::max<uint64_t>(bytes, 4), | |
| 88 | .usage = device::BufferUsage::Interop, | ||
| 89 | ✗ | .label = "dlpack copy"}); | |
| 90 | ✗ | if (bytes > 0) { | |
| 91 | ✗ | device::Queue& queue = device.GetQueue(); | |
| 92 | ✗ | device::CommandEncoder encoder = queue.CreateEncoder(); | |
| 93 | ✗ | encoder.Copy(tensor.GetBuffer().Slice(tensor.Offset(), bytes), copied->Slice(0, bytes)); | |
| 94 | ✗ | queue.Submit(encoder.Finish()); | |
| 95 | ✗ | } | |
| 96 | ✗ | return device::Tensor(copied, 0, tensor.Type(), {shape.begin(), shape.end()}, | |
| 97 | ✗ | {strides.begin(), strides.end()}); | |
| 98 | ✗ | } | |
| 99 | |||
| 100 | ✗ | void CheckStream(int type, int64_t stream) { | |
| 101 | ✗ | if (type == kDLCUDA && stream == 0) { | |
| 102 | ✗ | throw nb::value_error("CUDA consumers name the default stream as 1 or 2, never 0"); | |
| 103 | } | ||
| 104 | ✗ | if (type == kDLROCM && (stream == 1 || stream == 2)) { | |
| 105 | ✗ | throw nb::value_error("ROCm consumers name the default stream as 0, never 1 or 2"); | |
| 106 | } | ||
| 107 | ✗ | } | |
| 108 | |||
| 109 | ✗ | PyObject* VersionedCapsule(DLManagedTensorVersioned* managed) { | |
| 110 | ✗ | return PyCapsule_New(managed, kVersioned, [](PyObject* self) { | |
| 111 | ✗ | if (PyCapsule_IsValid(self, kVersioned)) { | |
| 112 | ✗ | auto* tensor = | |
| 113 | ✗ | static_cast<DLManagedTensorVersioned*>(PyCapsule_GetPointer(self, kVersioned)); | |
| 114 | ✗ | tensor->deleter(tensor); | |
| 115 | } | ||
| 116 | ✗ | }); | |
| 117 | } | ||
| 118 | |||
| 119 | ✗ | PyObject* UnversionedCapsule(DLManagedTensorVersioned* managed) { | |
| 120 | ✗ | auto* legacy = new DLManagedTensor(); | |
| 121 | ✗ | legacy->dl_tensor = managed->dl_tensor; | |
| 122 | ✗ | legacy->manager_ctx = managed; | |
| 123 | ✗ | legacy->deleter = [](DLManagedTensor* self) { | |
| 124 | ✗ | auto* inner = static_cast<DLManagedTensorVersioned*>(self->manager_ctx); | |
| 125 | ✗ | inner->deleter(inner); | |
| 126 | ✗ | delete self; | |
| 127 | ✗ | }; | |
| 128 | ✗ | return PyCapsule_New(legacy, kUnversioned, [](PyObject* self) { | |
| 129 | ✗ | if (PyCapsule_IsValid(self, kUnversioned)) { | |
| 130 | ✗ | auto* tensor = static_cast<DLManagedTensor*>(PyCapsule_GetPointer(self, kUnversioned)); | |
| 131 | ✗ | tensor->deleter(tensor); | |
| 132 | } | ||
| 133 | ✗ | }); | |
| 134 | } | ||
| 135 | |||
| 136 | 1170 | std::optional<int64_t> RequestedStream(nb::handle source, std::optional<uintptr_t> stream) { | |
| 137 | 1170 | if (!nb::hasattr(source, "__dlpack_device__")) { | |
| 138 | ✗ | return std::nullopt; | |
| 139 | } | ||
| 140 | 2340 | const nb::tuple device = nb::cast<nb::tuple>(source.attr("__dlpack_device__")()); | |
| 141 | 2340 | const int type = nb::cast<int>(device[0]); | |
| 142 | 1170 | const bool cuda = type == kDLCUDA || type == kDLCUDAHost || type == kDLCUDAManaged; | |
| 143 | 1170 | const bool rocm = type == kDLROCM || type == kDLROCMHost; | |
| 144 | 1170 | if (!cuda && !rocm) { | |
| 145 | 1170 | return std::nullopt; | |
| 146 | } | ||
| 147 | ✗ | if (stream && (*stream != 0 || !cuda)) { | |
| 148 | ✗ | return int64_t(*stream); | |
| 149 | } | ||
| 150 | ✗ | return cuda ? 1 : 0; | |
| 151 | 1170 | } | |
| 152 | |||
| 153 | 1170 | nb::object RequestCapsule(nb::handle source, std::optional<int64_t> stream) { | |
| 154 | 1170 | const nb::object request = source.attr("__dlpack__"); | |
| 155 | 1170 | try { | |
| 156 | 2340 | return stream ? request("stream"_a = *stream, "max_version"_a = nb::make_tuple(1, 0)) | |
| 157 | 4680 | : request("max_version"_a = nb::make_tuple(1, 0)); | |
| 158 | ✗ | } catch (nb::python_error& error) { | |
| 159 | ✗ | if (!error.matches(PyExc_TypeError)) { | |
| 160 | ✗ | throw; | |
| 161 | } | ||
| 162 | ✗ | return stream ? request("stream"_a = *stream) : request(); | |
| 163 | ✗ | } | |
| 164 | 1170 | } | |
| 165 | |||
| 166 | } | ||
| 167 | |||
| 168 | 222 | DLDataType DataType(device::DType type) { | |
| 169 | 222 | const uint8_t bits = uint8_t(device::ElementBytes(type) * 8); | |
| 170 | 222 | switch (type) { | |
| 171 | 215 | case device::DType::Float32: | |
| 172 | 215 | case device::DType::Float64: | |
| 173 | 215 | return {kDLFloat, bits, 1}; | |
| 174 | ✗ | case device::DType::Int32: | |
| 175 | ✗ | case device::DType::Int64: | |
| 176 | ✗ | return {kDLInt, bits, 1}; | |
| 177 | 7 | default: | |
| 178 | 7 | return {kDLUInt, bits, 1}; | |
| 179 | } | ||
| 180 | } | ||
| 181 | |||
| 182 | 2 | nb::tuple TensorDevice(const device::Tensor& tensor) { | |
| 183 | 2 | if (tensor.Empty()) { | |
| 184 | ✗ | throw nb::buffer_error("An empty tensor has no memory to share"); | |
| 185 | } | ||
| 186 | 2 | if (const std::optional<DLDevice> shared = tensor.GetBuffer().GetDevice().SharedDevice()) { | |
| 187 | ✗ | return nb::make_tuple(int(shared->device_type), shared->device_id); | |
| 188 | } | ||
| 189 | 2 | return nb::make_tuple(int(kDLVulkan), 0); | |
| 190 | } | ||
| 191 | |||
| 192 | 2 | nb::object ExportTensor(const device::Tensor& tensor, std::optional<int64_t> stream, | |
| 193 | std::optional<nb::tuple> maxVersion, std::optional<nb::tuple> dlDevice, | ||
| 194 | std::optional<bool> copy) { | ||
| 195 | 2 | const nb::tuple own = TensorDevice(tensor); | |
| 196 | 4 | const int type = nb::cast<int>(own[0]); | |
| 197 | 4 | const int id = nb::cast<int>(own[1]); | |
| 198 | 2 | bool toHost = false; | |
| 199 | 2 | if (dlDevice) { | |
| 200 | ✗ | const int wanted = nb::cast<int>((*dlDevice)[0]); | |
| 201 | ✗ | const int wantedId = nb::cast<int>((*dlDevice)[1]); | |
| 202 | ✗ | toHost = wanted == kDLCPU; | |
| 203 | ✗ | if (toHost && copy == false) { | |
| 204 | ✗ | throw nb::buffer_error("This tensor reaches the host only as a copy, and copy=False " | |
| 205 | ✗ | "forbids one"); | |
| 206 | } | ||
| 207 | ✗ | if (!toHost && (wanted != type || wantedId != id)) { | |
| 208 | ✗ | throw nb::buffer_error(fmt::format("This tensor is on device ({}, {}); it's exported " | |
| 209 | "there or copied to the host, not to ({}, {})", | ||
| 210 | type, id, wanted, wantedId) | ||
| 211 | ✗ | .c_str()); | |
| 212 | } | ||
| 213 | } | ||
| 214 | 2 | if (!toHost && type == kDLVulkan) { | |
| 215 | 2 | throw nb::buffer_error("This tensor's memory isn't shared with CUDA or HIP; ask for " | |
| 216 | 2 | "dl_device=(1, 0) to copy it to the host"); | |
| 217 | } | ||
| 218 | ✗ | if (!toHost && stream) { | |
| 219 | ✗ | CheckStream(type, *stream); | |
| 220 | } | ||
| 221 | |||
| 222 | ✗ | DLManagedTensorVersioned* managed = nullptr; | |
| 223 | ✗ | try { | |
| 224 | ✗ | nb::gil_scoped_release release; | |
| 225 | ✗ | managed = CreateExport(copy == true && !toHost ? CopyOnDevice(tensor) : tensor, toHost); | |
| 226 | ✗ | if (copy == true) { | |
| 227 | ✗ | managed->flags |= DLPACK_FLAG_BITMASK_IS_COPIED; | |
| 228 | } | ||
| 229 | ✗ | if (!toHost && stream != -1) { | |
| 230 | ✗ | const int64_t handle = stream.value_or(type == kDLCUDA ? 1 : 0); | |
| 231 | ✗ | tensor.GetBuffer().GetDevice().GetQueue().Mark().Wait( | |
| 232 | Stream::Compute(reinterpret_cast<void*>(uintptr_t(handle)))); | ||
| 233 | } | ||
| 234 | ✗ | } catch (const UnsupportedError& error) { | |
| 235 | ✗ | if (managed != nullptr) { | |
| 236 | ✗ | managed->deleter(managed); | |
| 237 | } | ||
| 238 | ✗ | throw nb::buffer_error(error.what()); | |
| 239 | ✗ | } catch (...) { | |
| 240 | ✗ | if (managed != nullptr) { | |
| 241 | ✗ | managed->deleter(managed); | |
| 242 | } | ||
| 243 | ✗ | throw; | |
| 244 | ✗ | } | |
| 245 | 2 | const bool versioned = maxVersion && nb::cast<int>((*maxVersion)[0]) >= 1; | |
| 246 | ✗ | PyObject* capsule = versioned ? VersionedCapsule(managed) : UnversionedCapsule(managed); | |
| 247 | ✗ | if (capsule == nullptr) { | |
| 248 | ✗ | managed->deleter(managed); | |
| 249 | ✗ | throw nb::python_error(); | |
| 250 | } | ||
| 251 | ✗ | return nb::steal(capsule); | |
| 252 | ✗ | } | |
| 253 | |||
| 254 | 1170 | TakenTensor TakeTensor(nb::handle source, std::optional<uintptr_t> stream) { | |
| 255 | 1170 | const std::optional<int64_t> requested = RequestedStream(source, stream); | |
| 256 | 1170 | const nb::object capsule = RequestCapsule(source, requested); | |
| 257 | 1170 | auto* producer = new Producer(); | |
| 258 | 1170 | if (PyCapsule_IsValid(capsule.ptr(), kVersioned)) { | |
| 259 | 1170 | producer->versioned = | |
| 260 | 1170 | static_cast<DLManagedTensorVersioned*>(PyCapsule_GetPointer(capsule.ptr(), kVersioned)); | |
| 261 | 1170 | PyCapsule_SetName(capsule.ptr(), "used_dltensor_versioned"); | |
| 262 | 1170 | producer->wrapper.flags = producer->versioned->flags; | |
| 263 | 1170 | producer->wrapper.dl_tensor = producer->versioned->dl_tensor; | |
| 264 | ✗ | } else if (PyCapsule_IsValid(capsule.ptr(), kUnversioned)) { | |
| 265 | ✗ | producer->unversioned = | |
| 266 | ✗ | static_cast<DLManagedTensor*>(PyCapsule_GetPointer(capsule.ptr(), kUnversioned)); | |
| 267 | ✗ | PyCapsule_SetName(capsule.ptr(), "used_dltensor"); | |
| 268 | ✗ | producer->wrapper.dl_tensor = producer->unversioned->dl_tensor; | |
| 269 | } else { | ||
| 270 | ✗ | delete producer; | |
| 271 | ✗ | throw nb::type_error("__dlpack__ returned no DLPack capsule"); | |
| 272 | } | ||
| 273 | |||
| 274 | 1170 | producer->wrapper.version = {DLPACK_MAJOR_VERSION, DLPACK_MINOR_VERSION}; | |
| 275 | 1170 | producer->wrapper.manager_ctx = producer; | |
| 276 | 1170 | producer->wrapper.deleter = ReleaseProducer; | |
| 277 | 1170 | const Stream followed = requested | |
| 278 | 1170 | ? Stream::Compute(reinterpret_cast<void*>(uintptr_t(*requested))) | |
| 279 | 1170 | : ToStream(stream); | |
| 280 | 1170 | return {&producer->wrapper, followed}; | |
| 281 | 1170 | } | |
| 282 | |||
| 283 | } | ||
| 284 |