stage/views/attribute.cpp
| Line | Branch | Exec | Source |
|---|---|---|---|
| 1 | // SPDX-License-Identifier: BSD-3-Clause | ||
| 2 | #include "chrono_batch_render/stage/views/attribute.h" | ||
| 3 | #include "chrono_batch_render/stage/views/view_state.h" | ||
| 4 | |||
| 5 | #include <algorithm> | ||
| 6 | #include <cstring> | ||
| 7 | #include <memory> | ||
| 8 | #include <utility> | ||
| 9 | |||
| 10 | #include <fmt/format.h> | ||
| 11 | |||
| 12 | #include "chrono_batch_render/device/buffer.h" | ||
| 13 | #include "chrono_batch_render/device/commands.h" | ||
| 14 | #include "chrono_batch_render/device/queue.h" | ||
| 15 | #include "chrono_batch_render/scene/schema/columns.h" | ||
| 16 | #include "chrono_batch_render/stage/columns/column_store.h" | ||
| 17 | #include "chrono_batch_render/stage/columns/kernels/cells.h" | ||
| 18 | #include "chrono_batch_render/stage/structure/packing.h" | ||
| 19 | #include "chrono_batch_render/stage/views/layout.h" | ||
| 20 | #include "chrono_batch_render/stage/views/staged.h" | ||
| 21 | #include "chrono_batch_render/util/profiler.h" | ||
| 22 | |||
| 23 | namespace chrono::batch_render { | ||
| 24 | |||
| 25 | namespace { | ||
| 26 | |||
| 27 | 5 | DLTensor Describe(const device::Tensor& tensor) { | |
| 28 | 5 | const device::DType type = tensor.Type(); | |
| 29 | 5 | DLTensor described{}; | |
| 30 | 5 | described.ndim = int32_t(tensor.Shape().size()); | |
| 31 | 10 | described.dtype = { | |
| 32 | 5 | uint8_t(type == device::DType::Float32 || type == device::DType::Float64 ? kDLFloat | |
| 33 | ✗ | : type == device::DType::Int32 || type == device::DType::Int64 ? kDLInt | |
| 34 | : kDLUInt), | ||
| 35 | 5 | uint8_t(device::ElementBytes(type) * 8), 1}; | |
| 36 | 5 | described.shape = const_cast<int64_t*>(tensor.Shape().data()); | |
| 37 | 5 | described.strides = | |
| 38 | 10 | tensor.Strides().empty() ? nullptr : const_cast<int64_t*>(tensor.Strides().data()); | |
| 39 | 5 | return described; | |
| 40 | } | ||
| 41 | |||
| 42 | ✗ | bool IsByteBool(const DLTensor& tensor) { | |
| 43 | ✗ | return tensor.ndim == 1 && tensor.dtype.bits == 8 && tensor.dtype.lanes == 1 && | |
| 44 | ✗ | (tensor.dtype.code == kDLBool || tensor.dtype.code == kDLUInt || | |
| 45 | ✗ | tensor.dtype.code == kDLInt); | |
| 46 | } | ||
| 47 | |||
| 48 | 4 | bool Selected(std::span<const uint64_t> mask, uint32_t index) { | |
| 49 | 4 | return mask.empty() || (mask[index / 64] >> (index % 64) & 1) != 0; | |
| 50 | } | ||
| 51 | |||
| 52 | 42179 | bool PicksSchema(ColumnRef column) { | |
| 53 | 42179 | return column.schema == SchemaId::Shader && column.index == kShaderId.index; | |
| 54 | } | ||
| 55 | |||
| 56 | 60 | void CheckHostWritten(ColumnRef column) { | |
| 57 | 60 | if (PicksSchema(column)) { | |
| 58 | ✗ | throw Error("A shader's info:id picks its node, so it's written from the host"); | |
| 59 | } | ||
| 60 | 60 | } | |
| 61 | |||
| 62 | 24 | std::vector<SchemaId> Nodes(const Structure& structure, Span<const std::byte> cells) { | |
| 63 | 24 | std::vector<SchemaId> nodes(cells.size() / sizeof(uint32_t)); | |
| 64 | 46 | for (size_t i = 0; i < nodes.size(); ++i) { | |
| 65 | 24 | uint32_t token = 0; | |
| 66 | 24 | std::memcpy(&token, cells.data() + i * sizeof(uint32_t), sizeof(uint32_t)); | |
| 67 | 24 | const std::string_view id = structure.GetTokens().Text(Token{token}); | |
| 68 | 24 | nodes[i] = ShaderSchema(id); | |
| 69 | 24 | if (nodes[i] == SchemaId::None) { | |
| 70 | 2 | throw Error("No shader has info:id \"{}\"", id); | |
| 71 | } | ||
| 72 | } | ||
| 73 | 22 | return nodes; | |
| 74 | 2 | } | |
| 75 | |||
| 76 | 12 | std::vector<int64_t> Shape(uint32_t count, const ColumnLayout& layout) { | |
| 77 | 12 | std::vector<int64_t> shape = {int64_t(count)}; | |
| 78 | 12 | if (layout.lanes > 1) { | |
| 79 | 1 | shape.push_back(layout.lanes); | |
| 80 | } | ||
| 81 | 12 | return shape; | |
| 82 | ✗ | } | |
| 83 | |||
| 84 | } | ||
| 85 | |||
| 86 | 171887 | UntypedAttribute::UntypedAttribute(std::shared_ptr<ViewState> view, ColumnRef column) | |
| 87 | 171887 | : m_view(std::move(view)), m_column(column) { | |
| 88 | 171887 | if (!m_column) { | |
| 89 | ✗ | throw Error("An attribute of no column"); | |
| 90 | } | ||
| 91 | 179010 | if (m_view->Schema() != SchemaId::None && !IsA(m_view->Schema(), m_column.schema) && | |
| 92 | 7123 | !IsApi(m_column.schema)) { | |
| 93 | 9 | throw Error("A view of {} has no {}'s {}", Info(m_view->Schema()).name, | |
| 94 | 9 | Info(m_column.schema).name, m_column.column->field); | |
| 95 | } | ||
| 96 | 171887 | } | |
| 97 | |||
| 98 | ✗ | void UntypedAttribute::CheckValues(size_t count) const { | |
| 99 | ✗ | if (count != Size()) { | |
| 100 | ✗ | throw Error("{} values for an attribute of {} prims", count, Size()); | |
| 101 | } | ||
| 102 | ✗ | } | |
| 103 | |||
| 104 | 16249 | ColumnRef UntypedAttribute::Checked(ColumnRef column, bool held, bool arrays) { | |
| 105 | 16249 | if (!held) { | |
| 106 | 6 | throw Error("{}'s {} isn't {} the type asked for", Info(column.schema).name, | |
| 107 | 5 | column.column->field, arrays ? "an array of" : "held as"); | |
| 108 | } | ||
| 109 | 16247 | return column; | |
| 110 | } | ||
| 111 | |||
| 112 | 131497 | uint32_t UntypedAttribute::Size() const { | |
| 113 | 131497 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 114 | 131497 | return m_view->Size(); | |
| 115 | 131497 | } | |
| 116 | |||
| 117 | 220517 | const ViewRows& UntypedAttribute::Rows() const { | |
| 118 | 220517 | return m_view->Rows(m_column.schema); | |
| 119 | } | ||
| 120 | |||
| 121 | 12726 | bool UntypedAttribute::Dense() const { | |
| 122 | 12726 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 123 | 25452 | return Rows().Dense(); | |
| 124 | 12726 | } | |
| 125 | |||
| 126 | 418991 | ColumnStore& UntypedAttribute::GetColumnStore() const { | |
| 127 | 418991 | return m_view->GetColumnStore(); | |
| 128 | } | ||
| 129 | |||
| 130 | 12734 | ColumnTable& UntypedAttribute::GetTable() const { | |
| 131 | 12734 | return GetColumnStore().Schema(m_column.schema); | |
| 132 | } | ||
| 133 | |||
| 134 | 42186 | uint32_t UntypedAttribute::CellBytes() const { | |
| 135 | 42186 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 136 | 84372 | return ElementBytes(m_column.column->type); | |
| 137 | 42186 | } | |
| 138 | |||
| 139 | 43213 | void UntypedAttribute::CheckWritable(bool arrays) const { | |
| 140 | 43213 | if (Has(m_column.column->flags, ColumnFlags::Internal)) { | |
| 141 | ✗ | throw Error("{}'s {} is internal; the structure writes it", Info(m_column.schema).name, | |
| 142 | ✗ | m_column.column->field); | |
| 143 | } | ||
| 144 | 43213 | if (!arrays && IsArray(m_column.column->type)) { | |
| 145 | 1 | throw UnsupportedError("{}'s {} holds arrays, which are written per prim", | |
| 146 | 2 | Info(m_column.schema).name, m_column.column->field); | |
| 147 | } | ||
| 148 | 43212 | } | |
| 149 | |||
| 150 | 43185 | void UntypedAttribute::CheckNotAttached() const { | |
| 151 | 43187 | if (GetColumnStore().AttachmentCount() > 0 && | |
| 152 | 2 | GetColumnStore().Attached(m_column, Rows().rows)) { | |
| 153 | 2 | throw Error("{}'s {} is attached; write the attached tensor and touch the attribute", | |
| 154 | 4 | Info(m_column.schema).name, m_column.column->field); | |
| 155 | } | ||
| 156 | 43183 | } | |
| 157 | |||
| 158 | 77284 | void UntypedAttribute::StampWorlds() const { | |
| 159 | 154576 | for (const uint32_t world : Rows().worlds) { | |
| 160 | 77292 | GetColumnStore().StampWorld(world); | |
| 161 | } | ||
| 162 | 77284 | } | |
| 163 | |||
| 164 | 42051 | void UntypedAttribute::Fill(Span<const std::byte> cell) { | |
| 165 | 42051 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 166 | 42051 | CheckWritable(); | |
| 167 | 42050 | CheckNotAttached(); | |
| 168 | 42049 | if (cell.size() != CellBytes()) { | |
| 169 | ✗ | throw Error("{} bytes for {}, whose cells are {}", cell.size(), m_column.column->field, | |
| 170 | ✗ | CellBytes()); | |
| 171 | } | ||
| 172 | 42049 | const std::vector<SchemaId> nodes = | |
| 173 | 42049 | PicksSchema(m_column) ? Nodes(m_view->GetStructure(), cell) : std::vector<SchemaId>{}; | |
| 174 | 42047 | const ViewRows& rows = Rows(); | |
| 175 | 84100 | for (const ViewBlock& block : rows.blocks) { | |
| 176 | 42053 | GetColumnStore().Fill(m_column, block.firstRow, block.count, cell); | |
| 177 | } | ||
| 178 | 42047 | StampWorlds(); | |
| 179 | 42047 | Retype(nodes); | |
| 180 | 42051 | } | |
| 181 | |||
| 182 | 70 | void UntypedAttribute::Write(Span<const std::byte> cells) { | |
| 183 | 70 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 184 | 70 | CHRONO_BATCH_RENDER_ZONE("Scene/Write"); | |
| 185 | 70 | CheckWritable(); | |
| 186 | 70 | CheckNotAttached(); | |
| 187 | 70 | const std::vector<SchemaId> nodes = | |
| 188 | 70 | PicksSchema(m_column) ? Nodes(m_view->GetStructure(), cells) : std::vector<SchemaId>{}; | |
| 189 | 70 | StageCells(cells); | |
| 190 | 70 | Retype(nodes); | |
| 191 | 72 | } | |
| 192 | |||
| 193 | 42115 | void UntypedAttribute::Retype(Span<const SchemaId> nodes) { | |
| 194 | 42115 | if (nodes.empty()) { | |
| 195 | 42093 | return; | |
| 196 | } | ||
| 197 | 22 | const std::vector<uint32_t> prims(m_view->Prims().begin(), m_view->Prims().end()); | |
| 198 | 44 | for (uint32_t index = 0; index < prims.size(); ++index) { | |
| 199 | 22 | if (m_view->Live(index)) { | |
| 200 | 22 | m_view->GetStructure().Retype(prims[index], nodes[nodes.size() == 1 ? 0 : index]); | |
| 201 | } | ||
| 202 | } | ||
| 203 | 22 | } | |
| 204 | |||
| 205 | 78 | void UntypedAttribute::StageCells(Span<const std::byte> cells) { | |
| 206 | 78 | const uint32_t cell = CellBytes(); | |
| 207 | 78 | if (cells.size() != size_t(cell) * Size()) { | |
| 208 | 8 | throw Error("{} bytes for {} prims' {}, whose cells are {}", cells.size(), Size(), | |
| 209 | 2 | m_column.column->field, cell); | |
| 210 | } | ||
| 211 | 76 | const ViewRows& rows = Rows(); | |
| 212 | 360 | for (const ViewBlock& block : rows.blocks) { | |
| 213 | 284 | GetColumnStore().Stage( | |
| 214 | 284 | m_column, block.firstRow, | |
| 215 | 284 | cells.subspan(size_t(block.first) * cell, size_t(block.count) * cell)); | |
| 216 | } | ||
| 217 | 76 | StampWorlds(); | |
| 218 | 76 | } | |
| 219 | |||
| 220 | 3 | device::Operation UntypedAttribute::Write(const DLTensor& source, Stream producer, | |
| 221 | const DLTensor* mask) { | ||
| 222 | 3 | const StoreLock* store = m_view ? m_view->GetColumnStore().Lock() : nullptr; | |
| 223 | 3 | std::optional<StoreLock::Guard> lock = | |
| 224 | 3 | store ? store->TryLock() : std::optional<StoreLock::Guard>(std::in_place); | |
| 225 | 3 | if (!lock) { | |
| 226 | 1 | device::Device& device = GetColumnStore().GetDevice(); | |
| 227 | 1 | std::optional<StagedTensor> values; | |
| 228 | 1 | std::optional<StagedTensor> staged; | |
| 229 | 2 | if (store->CanDefer() && StageOptional(device, &source, producer, values) && | |
| 230 | 1 | StageOptional(device, mask, producer, staged)) { | |
| 231 | 2 | store->Defer([attribute = *this, values = std::move(*values), | |
| 232 | staged = std::move(staged)]() mutable { | ||
| 233 | 1 | DLTensor view; | |
| 234 | 1 | const DLTensor written = values.View(); | |
| 235 | 1 | attribute.Write(written, values.Producer(), ViewOptional(staged, view)); | |
| 236 | 1 | }); | |
| 237 | 1 | return {}; | |
| 238 | } | ||
| 239 | ✗ | lock = store->Lock(); | |
| 240 | 1 | } | |
| 241 | 2 | CHRONO_BATCH_RENDER_ZONE("Scene/Write"); | |
| 242 | 2 | CheckWritable(); | |
| 243 | 2 | CheckHostWritten(m_column); | |
| 244 | 2 | CheckNotAttached(); | |
| 245 | 2 | return WriteTensor(source, producer, mask); | |
| 246 | 5 | } | |
| 247 | |||
| 248 | 9 | device::Operation UntypedAttribute::WriteTensor(const DLTensor& source, Stream producer, | |
| 249 | const DLTensor* mask) { | ||
| 250 | 9 | if (Size() == 0) { | |
| 251 | ✗ | return {}; | |
| 252 | } | ||
| 253 | 9 | const ColumnInfo& column = *m_column.column; | |
| 254 | 9 | const std::byte* data = TensorBytes(source); | |
| 255 | 9 | if (OnHost(source.device)) { | |
| 256 | 9 | WaitOnHost(GetColumnStore().GetDevice(), producer); | |
| 257 | 9 | std::vector<uint64_t> words; | |
| 258 | 9 | if (mask) { | |
| 259 | 1 | if (!OnHost(mask->device)) { | |
| 260 | ✗ | throw UnsupportedError("A host tensor is written with a host mask"); | |
| 261 | } | ||
| 262 | 1 | const uint64_t* bits = reinterpret_cast<const uint64_t*>(TensorBytes(*mask)); | |
| 263 | 1 | words.assign(bits, bits + (Size() + 63) / 64); | |
| 264 | } | ||
| 265 | 9 | std::vector<std::byte> cells; | |
| 266 | 9 | if (column.type == ColumnType::Bool && IsByteBool(source) && source.shape[0] == Size()) { | |
| 267 | ✗ | std::vector<uint32_t> words32(Size()); | |
| 268 | ✗ | for (uint32_t i = 0; i < Size(); ++i) { | |
| 269 | ✗ | words32[i] = data[i] != std::byte{0} ? 1 : 0; | |
| 270 | } | ||
| 271 | ✗ | const std::span<const std::byte> bytes = std::as_bytes(std::span(words32)); | |
| 272 | ✗ | cells.assign(bytes.begin(), bytes.end()); | |
| 273 | ✗ | } else { | |
| 274 | 9 | if (TensorRows(source, Layout(column.type), column.field) != Size()) { | |
| 275 | ✗ | throw Error("A tensor of {} rows for {} prims' {}", source.shape[0], Size(), | |
| 276 | ✗ | column.field); | |
| 277 | } | ||
| 278 | 9 | cells.assign(data, data + size_t(CellBytes()) * Size()); | |
| 279 | } | ||
| 280 | 9 | if (words.empty()) { | |
| 281 | 8 | StageCells(cells); | |
| 282 | 8 | return {}; | |
| 283 | } | ||
| 284 | 1 | const uint32_t cell = CellBytes(); | |
| 285 | 1 | const ViewRows& rows = Rows(); | |
| 286 | 5 | for (uint32_t i = 0; i < Size(); ++i) { | |
| 287 | 4 | if (rows.rows[i] != kNoRow && Selected(words, i)) { | |
| 288 | 2 | GetColumnStore().Stage(m_column, rows.rows[i], | |
| 289 | 2 | std::span(cells).subspan(size_t(i) * cell, cell)); | |
| 290 | } | ||
| 291 | } | ||
| 292 | 1 | StampWorlds(); | |
| 293 | 1 | return {}; | |
| 294 | 9 | } | |
| 295 | |||
| 296 | ✗ | if (TensorRows(source, Layout(column.type), column.field) != Size()) { | |
| 297 | ✗ | throw Error("A tensor of {} rows for {} prims' {}", source.shape[0], Size(), column.field); | |
| 298 | } | ||
| 299 | ✗ | device::Device& device = GetColumnStore().GetDevice(); | |
| 300 | ✗ | CheckReadable(device, source.device); | |
| 301 | ✗ | uint64_t cells = uint64_t(data); | |
| 302 | ✗ | Ref<device::Buffer> staging; | |
| 303 | ✗ | if (!device.ReadsDirectly(source.device)) { | |
| 304 | ✗ | staging = | |
| 305 | ✗ | device.Stage(cells, uint64_t(CellBytes()) * Size(), producer, "attribute staging"); | |
| 306 | ✗ | cells = staging->Address(); | |
| 307 | } | ||
| 308 | ✗ | return Scatter(cells, producer, Upload(mask, producer)); | |
| 309 | ✗ | } | |
| 310 | |||
| 311 | 6 | device::Operation UntypedAttribute::Write(const device::Tensor& source, Stream producer, | |
| 312 | const device::Tensor& mask) { | ||
| 313 | 6 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 314 | 6 | CHRONO_BATCH_RENDER_ZONE("Scene/Write"); | |
| 315 | 6 | CheckWritable(); | |
| 316 | 6 | CheckHostWritten(m_column); | |
| 317 | 6 | CheckNotAttached(); | |
| 318 | 6 | if (Size() == 0) { | |
| 319 | ✗ | return {}; | |
| 320 | } | ||
| 321 | 6 | const ColumnInfo& column = *m_column.column; | |
| 322 | 6 | device::Device& device = GetColumnStore().GetDevice(); | |
| 323 | 6 | if (source.Empty() || &source.GetDevice() != &device) { | |
| 324 | 1 | throw Error("{} is written from tensors on the scene's {}; Device::Copy or " | |
| 325 | "Device::Import moves them", | ||
| 326 | 1 | column.field, device.Adapter().alias); | |
| 327 | } | ||
| 328 | 5 | if (TensorRows(Describe(source), Layout(column.type), column.field) != Size()) { | |
| 329 | ✗ | throw Error("A tensor of {} rows for {} prims' {}", source.Shape()[0], Size(), | |
| 330 | ✗ | column.field); | |
| 331 | } | ||
| 332 | 8 | return Scatter(source.Address(), producer, Upload(mask)); | |
| 333 | 8 | } | |
| 334 | |||
| 335 | 38 | UntypedAttribute::Mask UntypedAttribute::Upload(const DLTensor* mask, Stream producer) const { | |
| 336 | 38 | if (!mask) { | |
| 337 | 38 | return {}; | |
| 338 | } | ||
| 339 | ✗ | const uint64_t words = (uint64_t(Size()) + 63) / 64; | |
| 340 | ✗ | if (mask->ndim != 1 || mask->dtype.code != kDLUInt || mask->dtype.bits != 64 || | |
| 341 | ✗ | uint64_t(mask->shape[0]) < words) { | |
| 342 | ✗ | throw Error("A mask is a [words] tensor of uint64 with a bit per prim; {} prims take {} " | |
| 343 | "words", | ||
| 344 | ✗ | Size(), words); | |
| 345 | } | ||
| 346 | ✗ | device::Device& device = GetColumnStore().GetDevice(); | |
| 347 | ✗ | const std::byte* bits = TensorBytes(*mask); | |
| 348 | ✗ | Mask uploaded; | |
| 349 | ✗ | if (OnHost(mask->device)) { | |
| 350 | ✗ | WaitOnHost(device, producer); | |
| 351 | ✗ | uploaded.buffer = device.CreateBuffer( | |
| 352 | {.label = "attribute mask"}, | ||
| 353 | ✗ | std::span<const uint64_t>(reinterpret_cast<const uint64_t*>(bits), words)); | |
| 354 | ✗ | } else if (device.ReadsDirectly(mask->device)) { | |
| 355 | ✗ | uploaded.address = uint64_t(bits); | |
| 356 | ✗ | return uploaded; | |
| 357 | } else { | ||
| 358 | ✗ | CheckReadable(device, mask->device); | |
| 359 | ✗ | uploaded.buffer = | |
| 360 | ✗ | device.Stage(uint64_t(bits), words * sizeof(uint64_t), producer, "attribute mask"); | |
| 361 | } | ||
| 362 | ✗ | uploaded.address = uploaded.buffer->Address(); | |
| 363 | ✗ | return uploaded; | |
| 364 | ✗ | } | |
| 365 | |||
| 366 | 35122 | UntypedAttribute::Mask UntypedAttribute::Upload(const device::Tensor& mask) const { | |
| 367 | 35122 | if (mask.Empty()) { | |
| 368 | 35118 | return {}; | |
| 369 | } | ||
| 370 | 4 | const uint64_t words = (uint64_t(Size()) + 63) / 64; | |
| 371 | 8 | if (mask.Shape().size() != 1 || mask.Type() != device::DType::UInt64 || | |
| 372 | 12 | uint64_t(mask.Shape()[0]) < words || | |
| 373 | 8 | (!mask.Strides().empty() && mask.Shape()[0] > 1 && mask.Strides()[0] != 1)) { | |
| 374 | ✗ | throw Error("A mask is a contiguous [words] tensor of uint64 with a bit per prim; {} prims " | |
| 375 | "take {} words", | ||
| 376 | ✗ | Size(), words); | |
| 377 | } | ||
| 378 | 4 | device::Device& device = GetColumnStore().GetDevice(); | |
| 379 | 4 | if (&mask.GetDevice() != &device) { | |
| 380 | ✗ | throw Error("A mask on {} for a scene on {}; Device::Copy or Device::Import moves it", | |
| 381 | ✗ | mask.GetDevice().Adapter().alias, device.Adapter().alias); | |
| 382 | } | ||
| 383 | 8 | return {mask.Address(), {}}; | |
| 384 | 4 | } | |
| 385 | |||
| 386 | 7 | device::Operation UntypedAttribute::Scatter(uint64_t cells, Stream producer, const Mask& selected) { | |
| 387 | 7 | ColumnStore& store = GetColumnStore(); | |
| 388 | 7 | store.Flush(); | |
| 389 | 7 | store.GetDevice().GetQueue().Follow(producer); | |
| 390 | 7 | CellsParams params{}; | |
| 391 | 7 | params.target = store.Stamps(m_column); | |
| 392 | 7 | params.column.address = GetTable().Pages(m_column.index); | |
| 393 | 7 | params.cells.address = cells; | |
| 394 | 7 | const RowSource rowSource = m_view->Source(m_column.schema); | |
| 395 | 7 | params.rows.address = rowSource.rows; | |
| 396 | 7 | params.firstRow = rowSource.firstRow; | |
| 397 | 7 | params.mask.address = selected.address; | |
| 398 | 7 | params.count = Size(); | |
| 399 | 7 | params.words = CellBytes() / sizeof(uint32_t); | |
| 400 | 7 | store.Touch(m_column); | |
| 401 | 7 | StampWorlds(); | |
| 402 | 7 | const device::Operation scattered = | |
| 403 | 7 | store.Dispatch("ScatterCells", params, uint64_t(params.count) * params.words); | |
| 404 | 7 | store.Reassert(m_column, Rows().rows); | |
| 405 | 7 | return scattered; | |
| 406 | } | ||
| 407 | |||
| 408 | 35 | device::Operation UntypedAttribute::Stamp(Stream producer, const DLTensor* mask) { | |
| 409 | 35 | ColumnStore& store = GetColumnStore(); | |
| 410 | 35 | store.GetDevice().GetQueue().Follow(producer); | |
| 411 | 35 | const Mask selected = Upload(mask, producer); | |
| 412 | 35 | StampParams params{}; | |
| 413 | 35 | params.target = store.Stamps(m_column); | |
| 414 | 35 | const RowSource rowSource = m_view->Source(m_column.schema); | |
| 415 | 35 | params.rows.address = rowSource.rows; | |
| 416 | 35 | params.firstRow = rowSource.firstRow; | |
| 417 | 35 | params.mask.address = selected.address; | |
| 418 | 35 | params.count = Size(); | |
| 419 | 35 | store.Touch(m_column); | |
| 420 | 35 | StampWorlds(); | |
| 421 | 35 | if (IsArray(m_column.column->type)) { | |
| 422 | 23 | store.LogArrayRows(m_column, Rows().rows); | |
| 423 | } | ||
| 424 | 35 | const device::Operation stamped = store.Dispatch("StampRows", params, params.count); | |
| 425 | 35 | store.Reassert(m_column, Rows().rows); | |
| 426 | 35 | return stamped; | |
| 427 | 35 | } | |
| 428 | |||
| 429 | 14 | device::Operation UntypedAttribute::MarkChanged(Stream producer, const DLTensor* mask) { | |
| 430 | 14 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 431 | 14 | CheckWritable(true); | |
| 432 | 14 | CheckHostWritten(m_column); | |
| 433 | 14 | if (Size() == 0) { | |
| 434 | ✗ | return {}; | |
| 435 | } | ||
| 436 | 14 | if (GetColumnStore().AttachmentCount() > 0) { | |
| 437 | 4 | const Attachment* attachment = GetColumnStore().FindAttachment(m_column, Rows().rows); | |
| 438 | 4 | if (attachment && !attachment->inPlace) { | |
| 439 | 4 | const DLTensor& source = *attachment->tensor; | |
| 440 | 4 | return IsArray(m_column.column->type) ? CopyArrays(source, producer, mask) | |
| 441 | 2 | : WriteTensor(source, producer, mask); | |
| 442 | } | ||
| 443 | } | ||
| 444 | 10 | return Stamp(producer, mask); | |
| 445 | 14 | } | |
| 446 | |||
| 447 | 2 | device::Operation UntypedAttribute::AttachArrays(const Attachment& attachment, | |
| 448 | const DLTensor& source, Stream producer, | ||
| 449 | AttachFrom from) { | ||
| 450 | 2 | const ColumnInfo& column = *m_column.column; | |
| 451 | 2 | const uint32_t count = TensorItems(source, Layout(column.type), Size(), column.field); | |
| 452 | 2 | if (column.type == ColumnType::PrimArray) { | |
| 453 | ✗ | m_view->GetStructure().MarkUntrackedTargets(); | |
| 454 | } | ||
| 455 | 2 | ColumnStore& store = GetColumnStore(); | |
| 456 | 2 | store.Flush(); | |
| 457 | 2 | ArrayPool& heap = store.Arrays(column.type); | |
| 458 | 2 | device::Device& device = store.GetDevice(); | |
| 459 | 2 | device::Queue& queue = device.GetQueue(); | |
| 460 | 2 | const uint64_t bytes = uint64_t(count) * heap.ItemBytes(); | |
| 461 | 2 | std::vector<ArrayRef> old; | |
| 462 | 7 | for (const uint32_t row : attachment.rows) { | |
| 463 | 5 | old.push_back(store.Array(m_column, row)); | |
| 464 | 5 | if (from == AttachFrom::Scene && old.back().count != count) { | |
| 465 | ✗ | throw Error("{}'s {} attaches {} items per prim from prims whose arrays hold {}", | |
| 466 | ✗ | Info(m_column.schema).name, column.field, count, old.back().count); | |
| 467 | } | ||
| 468 | } | ||
| 469 | 2 | if (!attachment.inPlace && !OnHost(source.device)) { | |
| 470 | ✗ | if (from == AttachFrom::Scene) { | |
| 471 | ✗ | throw UnsupportedError("{} copies scene values into host tensors and ones it reads " | |
| 472 | "directly, not into memory of device type {}", | ||
| 473 | ✗ | device.Adapter().alias, int(source.device.device_type)); | |
| 474 | } | ||
| 475 | ✗ | CheckReadable(device, source.device); | |
| 476 | } | ||
| 477 | 2 | std::vector<ArrayRef> fresh; | |
| 478 | 2 | if (attachment.inPlace) { | |
| 479 | ✗ | queue.Follow(producer); | |
| 480 | ✗ | fresh = heap.Borrow(device.Borrow(source.device, uint64_t(TensorBytes(source)), | |
| 481 | ✗ | bytes * Size(), fmt::format("attached {}", column.field)), | |
| 482 | ✗ | Size(), count); | |
| 483 | } else { | ||
| 484 | 4 | fresh = heap.AllocateRun(std::vector<uint32_t>(Size(), count)); | |
| 485 | } | ||
| 486 | 2 | if (from == AttachFrom::Scene) { | |
| 487 | ✗ | device::CommandEncoder encoder = queue.CreateEncoder(); | |
| 488 | ✗ | for (uint32_t i = 0; i < Size(); ++i) { | |
| 489 | ✗ | encoder.Copy(heap.Range(old[i]).Slice(0, bytes), heap.Range(fresh[i]).Slice(0, bytes)); | |
| 490 | } | ||
| 491 | ✗ | queue.Submit(encoder.Finish()); | |
| 492 | ✗ | } | |
| 493 | 7 | for (uint32_t i = 0; i < Size(); ++i) { | |
| 494 | 5 | store.SetArray(m_column, attachment.rows[i], fresh[i]); | |
| 495 | } | ||
| 496 | 2 | store.Flush(); | |
| 497 | 2 | if (from == AttachFrom::Scene) { | |
| 498 | ✗ | if (!attachment.inPlace) { | |
| 499 | ✗ | WaitOnHost(device, producer); | |
| 500 | ✗ | const device::BufferRange first = heap.Range(fresh.front()); | |
| 501 | ✗ | queue.Read(first.GetBuffer().Slice(first.Offset(), bytes * Size()), | |
| 502 | ✗ | const_cast<std::byte*>(TensorBytes(source))); | |
| 503 | } | ||
| 504 | ✗ | return queue.Mark(); | |
| 505 | } | ||
| 506 | 2 | if (attachment.inPlace) { | |
| 507 | ✗ | return Stamp(Stream::None(), nullptr); | |
| 508 | } | ||
| 509 | 2 | return CopyArrays(source, producer, nullptr); | |
| 510 | 2 | } | |
| 511 | |||
| 512 | 4 | device::Operation UntypedAttribute::CopyArrays(const DLTensor& source, Stream producer, | |
| 513 | const DLTensor* mask) { | ||
| 514 | 4 | ColumnStore& store = GetColumnStore(); | |
| 515 | 4 | store.Flush(); | |
| 516 | 4 | ArrayPool& heap = store.Arrays(m_column.column->type); | |
| 517 | 4 | const ArrayRef first = store.Array(m_column, Rows().rows.front()); | |
| 518 | 4 | const device::BufferRange range = heap.Range(first); | |
| 519 | 4 | const device::BufferRange run = | |
| 520 | 4 | range.GetBuffer().Slice(range.Offset(), uint64_t(first.count) * heap.ItemBytes() * Size()); | |
| 521 | 4 | if (OnHost(source.device)) { | |
| 522 | 4 | WaitOnHost(store.GetDevice(), producer); | |
| 523 | 4 | store.GetDevice().GetQueue().Write(run, TensorBytes(source)); | |
| 524 | } else { | ||
| 525 | ✗ | store.GetDevice().CopyIn(run, uint64_t(TensorBytes(source)), producer); | |
| 526 | } | ||
| 527 | 4 | return Stamp(producer, mask); | |
| 528 | } | ||
| 529 | |||
| 530 | ✗ | void UntypedAttribute::ReleaseArrays() { | |
| 531 | ✗ | ColumnStore& store = GetColumnStore(); | |
| 532 | ✗ | store.Flush(); | |
| 533 | ✗ | ArrayPool& heap = store.Arrays(m_column.column->type); | |
| 534 | ✗ | const std::vector<uint32_t>& rows = Rows().rows; | |
| 535 | ✗ | const ArrayRef first = store.Array(m_column, rows.front()); | |
| 536 | ✗ | const std::vector<ArrayRef> fresh = | |
| 537 | ✗ | heap.AllocateRun(std::vector<uint32_t>(rows.size(), first.count)); | |
| 538 | ✗ | const uint64_t bytes = uint64_t(first.count) * heap.ItemBytes() * rows.size(); | |
| 539 | ✗ | const device::BufferRange from = heap.Range(first); | |
| 540 | ✗ | const device::BufferRange to = heap.Range(fresh.front()); | |
| 541 | ✗ | device::Queue& queue = store.GetDevice().GetQueue(); | |
| 542 | ✗ | device::CommandEncoder encoder = queue.CreateEncoder(); | |
| 543 | ✗ | encoder.Copy(from.GetBuffer().Slice(from.Offset(), bytes), | |
| 544 | ✗ | to.GetBuffer().Slice(to.Offset(), bytes)); | |
| 545 | ✗ | queue.Submit(encoder.Finish()); | |
| 546 | ✗ | for (uint32_t i = 0; i < rows.size(); ++i) { | |
| 547 | ✗ | store.SetArray(m_column, rows[i], fresh[i]); | |
| 548 | } | ||
| 549 | ✗ | store.Flush(); | |
| 550 | ✗ | } | |
| 551 | |||
| 552 | ✗ | device::Operation UntypedAttribute::WaitForReads(Stream consumer) const { | |
| 553 | ✗ | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 554 | ✗ | if (!Attached()) { | |
| 555 | ✗ | throw Error("{}'s {} isn't attached, so nothing of it waits", Info(m_column.schema).name, | |
| 556 | ✗ | m_column.column->field); | |
| 557 | } | ||
| 558 | ✗ | const device::Operation read = GetColumnStore().GetDevice().GetQueue().Mark(); | |
| 559 | ✗ | read.Wait(consumer); | |
| 560 | ✗ | return read; | |
| 561 | ✗ | } | |
| 562 | |||
| 563 | 4 | bool UntypedAttribute::Attached() const { | |
| 564 | 4 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 565 | 6 | return GetColumnStore().AttachmentCount() > 0 && | |
| 566 | 6 | GetColumnStore().FindAttachment(m_column, Rows().rows); | |
| 567 | 4 | } | |
| 568 | |||
| 569 | 9 | device::Operation UntypedAttribute::Attach(DLManagedTensor* tensor, Stream producer, | |
| 570 | AttachFrom from) { | ||
| 571 | 9 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 572 | 11 | return AttachTensor(tensor ? &tensor->dl_tensor : nullptr, producer, from, [tensor] { | |
| 573 | 14 | return std::shared_ptr<const DLTensor>(&tensor->dl_tensor, [tensor](const DLTensor*) { | |
| 574 | 7 | if (tensor->deleter) { | |
| 575 | 7 | tensor->deleter(tensor); | |
| 576 | } | ||
| 577 | 7 | }); | |
| 578 | 7 | }); | |
| 579 | 9 | } | |
| 580 | |||
| 581 | 3 | device::Operation UntypedAttribute::Attach(DLManagedTensorVersioned* tensor, Stream producer, | |
| 582 | AttachFrom from) { | ||
| 583 | 3 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 584 | 3 | if (tensor && (tensor->flags & DLPACK_FLAG_BITMASK_IS_COPIED)) { | |
| 585 | 1 | throw Error("{}'s {} attaches a copy, so later writes to the tensor wouldn't reach the " | |
| 586 | "scene; export it without copying", | ||
| 587 | 2 | Info(m_column.schema).name, m_column.column->field); | |
| 588 | } | ||
| 589 | 2 | if (tensor && from == AttachFrom::Scene && (tensor->flags & DLPACK_FLAG_BITMASK_READ_ONLY)) { | |
| 590 | 1 | throw Error("A read-only tensor can't take {}'s scene values of {}", | |
| 591 | 2 | Info(m_column.schema).name, m_column.column->field); | |
| 592 | } | ||
| 593 | 1 | return AttachTensor(tensor ? &tensor->dl_tensor : nullptr, producer, from, [tensor] { | |
| 594 | 2 | return std::shared_ptr<const DLTensor>(&tensor->dl_tensor, [tensor](const DLTensor*) { | |
| 595 | 1 | if (tensor->deleter) { | |
| 596 | ✗ | tensor->deleter(tensor); | |
| 597 | } | ||
| 598 | 1 | }); | |
| 599 | 1 | }); | |
| 600 | 3 | } | |
| 601 | |||
| 602 | 10 | device::Operation UntypedAttribute::AttachTensor(const DLTensor* tensor, Stream producer, | |
| 603 | AttachFrom from, const Owner& own) { | ||
| 604 | 10 | CheckWritable(true); | |
| 605 | 10 | CheckHostWritten(m_column); | |
| 606 | 10 | const ColumnInfo& column = *m_column.column; | |
| 607 | 10 | if (tensor == nullptr) { | |
| 608 | ✗ | throw Error("{} attaches no tensor", column.field); | |
| 609 | } | ||
| 610 | 10 | if (Size() == 0) { | |
| 611 | ✗ | throw Error("An empty view attaches no {}", column.field); | |
| 612 | } | ||
| 613 | 10 | const ViewRows& rows = Rows(); | |
| 614 | 10 | if (rows.live != Size()) { | |
| 615 | ✗ | throw Error("A view with removed prims can't attach {}", column.field); | |
| 616 | } | ||
| 617 | 10 | ColumnStore& store = GetColumnStore(); | |
| 618 | 10 | if (store.Attached(m_column, rows.rows)) { | |
| 619 | 1 | throw Error("{}'s {} is already attached for some of these prims", | |
| 620 | 2 | Info(m_column.schema).name, column.field); | |
| 621 | } | ||
| 622 | 9 | const DLTensor& source = *tensor; | |
| 623 | 9 | device::Device& device = store.GetDevice(); | |
| 624 | 9 | device::Queue& queue = device.GetQueue(); | |
| 625 | 9 | Attachment attachment{m_column, rows.rows, {}, device.ReadsDirectly(source.device), {}}; | |
| 626 | 9 | device::Operation ready; | |
| 627 | 9 | if (IsArray(column.type)) { | |
| 628 | 2 | ready = AttachArrays(attachment, source, producer, from); | |
| 629 | 7 | } else if (TensorRows(source, Layout(column.type), column.field) != Size()) { | |
| 630 | 4 | throw Error("A tensor of {} rows attached for {} prims' {}", source.shape[0], Size(), | |
| 631 | 1 | column.field); | |
| 632 | 6 | } else if (attachment.inPlace) { | |
| 633 | ✗ | if (!Dense()) { | |
| 634 | ✗ | throw Error("{}'s {} is attached in place only for prims allocated together; define " | |
| 635 | "them with BatchDesc::attachable or clone them with CloneDesc::attachable", | ||
| 636 | ✗ | Info(m_column.schema).name, column.field); | |
| 637 | } | ||
| 638 | ✗ | const uint32_t first = rows.blocks.front().firstRow; | |
| 639 | ✗ | const uint64_t bytes = uint64_t(CellBytes()) * Size(); | |
| 640 | ✗ | Ref<device::Buffer> borrowed = | |
| 641 | ✗ | device.Borrow(source.device, uint64_t(TensorBytes(source)), bytes, | |
| 642 | ✗ | fmt::format("attached {}", column.field)); | |
| 643 | ✗ | ColumnTable& table = GetTable(); | |
| 644 | ✗ | table.Flush(); | |
| 645 | ✗ | queue.Follow(producer); | |
| 646 | ✗ | if (from == AttachFrom::Scene) { | |
| 647 | ✗ | device::CommandEncoder encoder = queue.CreateEncoder(); | |
| 648 | ✗ | encoder.Copy(*table.Contiguous(m_column.index, first, Size()), | |
| 649 | borrowed->Slice(0, bytes)); | ||
| 650 | ✗ | queue.Submit(encoder.Finish()); | |
| 651 | ✗ | } | |
| 652 | ✗ | try { | |
| 653 | ✗ | table.Redirect(m_column.index, first, Size(), borrowed); | |
| 654 | ✗ | } catch (const Error& error) { | |
| 655 | ✗ | throw Error("{}'s {} can't be attached in place: {}. Define the prims with " | |
| 656 | "BatchDesc::attachable or clone them with CloneDesc::attachable", | ||
| 657 | ✗ | Info(m_column.schema).name, column.field, error.what()); | |
| 658 | ✗ | } | |
| 659 | ✗ | ready = from == AttachFrom::Tensor ? Stamp(Stream::None(), nullptr) : queue.Mark(); | |
| 660 | 6 | } else if (from == AttachFrom::Scene) { | |
| 661 | 1 | if (!OnHost(source.device)) { | |
| 662 | ✗ | throw UnsupportedError("{} copies scene values into host tensors and ones it reads " | |
| 663 | "directly, not into memory of device type {}", | ||
| 664 | ✗ | device.Adapter().alias, int(source.device.device_type)); | |
| 665 | } | ||
| 666 | 1 | const std::vector<std::byte> cells = ReadHost(); | |
| 667 | 1 | WaitOnHost(device, producer); | |
| 668 | 1 | std::copy(cells.begin(), cells.end(), const_cast<std::byte*>(TensorBytes(source))); | |
| 669 | 1 | } else { | |
| 670 | 5 | if (!OnHost(source.device)) { | |
| 671 | ✗ | CheckReadable(device, source.device); | |
| 672 | } | ||
| 673 | 5 | ready = WriteTensor(source, producer, nullptr); | |
| 674 | } | ||
| 675 | 16 | attachment.tensor = own(); | |
| 676 | 8 | store.Attach(std::move(attachment)); | |
| 677 | 8 | if (from == AttachFrom::Scene) { | |
| 678 | 1 | ready.Wait(producer); | |
| 679 | } | ||
| 680 | 8 | return ready; | |
| 681 | 9 | } | |
| 682 | |||
| 683 | 8 | device::Operation UntypedAttribute::Detach(Stream consumer) { | |
| 684 | 8 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 685 | 8 | ColumnStore& store = GetColumnStore(); | |
| 686 | 8 | const ViewRows& rows = Rows(); | |
| 687 | 8 | Attachment attachment = store.Detach(m_column, rows.rows); | |
| 688 | 7 | device::Queue& queue = store.GetDevice().GetQueue(); | |
| 689 | 7 | if (attachment.inPlace && IsArray(m_column.column->type)) { | |
| 690 | ✗ | ReleaseArrays(); | |
| 691 | 7 | } else if (attachment.inPlace) { | |
| 692 | ✗ | ColumnTable& table = GetTable(); | |
| 693 | ✗ | const uint32_t first = rows.blocks.front().firstRow; | |
| 694 | ✗ | const Ref<device::Buffer> borrowed = table.Restore(m_column.index, first); | |
| 695 | ✗ | const device::BufferRange owned = *table.Contiguous(m_column.index, first, Size()); | |
| 696 | ✗ | device::CommandEncoder encoder = queue.CreateEncoder(); | |
| 697 | ✗ | encoder.Copy(borrowed->Slice(0, owned.Bytes()), owned); | |
| 698 | ✗ | queue.Submit(encoder.Finish()); | |
| 699 | ✗ | } | |
| 700 | 7 | const device::Operation ready = queue.Mark(); | |
| 701 | 7 | if (consumer.IsCompute()) { | |
| 702 | ✗ | ready.Wait(consumer); | |
| 703 | ✗ | queue.Retire(std::const_pointer_cast<DLTensor>(std::move(attachment.tensor)), ready); | |
| 704 | } else { | ||
| 705 | 7 | ready.Wait(); | |
| 706 | 7 | attachment.tensor.reset(); | |
| 707 | } | ||
| 708 | 7 | return ready; | |
| 709 | 8 | } | |
| 710 | |||
| 711 | 16 | Ref<device::Buffer> UntypedAttribute::GatherCells() const { | |
| 712 | 16 | ColumnStore& store = GetColumnStore(); | |
| 713 | 16 | store.Flush(); | |
| 714 | 16 | device::Device& device = store.GetDevice(); | |
| 715 | 16 | const Ref<device::Buffer> cells = | |
| 716 | 16 | device.CreateBuffer({.bytes = std::max<uint64_t>(uint64_t(CellBytes()) * Size(), 4), | |
| 717 | .usage = device::BufferUsage::Interop, | ||
| 718 | 16 | .label = "gathered attribute"}); | |
| 719 | 16 | device::Queue& queue = device.GetQueue(); | |
| 720 | 16 | device::CommandEncoder encoder = queue.CreateEncoder(); | |
| 721 | 16 | encoder.Fill(*cells); | |
| 722 | 16 | queue.Submit(encoder.Finish()); | |
| 723 | 16 | CellsParams params{}; | |
| 724 | 16 | params.column.address = GetTable().Pages(m_column.index); | |
| 725 | 16 | params.cells.address = cells->Address(); | |
| 726 | 16 | const RowSource rowSource = m_view->Source(m_column.schema); | |
| 727 | 16 | params.rows.address = rowSource.rows; | |
| 728 | 16 | params.firstRow = rowSource.firstRow; | |
| 729 | 16 | params.count = Size(); | |
| 730 | 16 | params.words = CellBytes() / sizeof(uint32_t); | |
| 731 | 16 | store.Dispatch("GatherCells", params, uint64_t(params.count) * params.words); | |
| 732 | 16 | return cells; | |
| 733 | 64 | } | |
| 734 | |||
| 735 | 12716 | device::Download UntypedAttribute::ReadHostAsync() const { | |
| 736 | 12716 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 737 | 12716 | if (Size() == 0) { | |
| 738 | ✗ | return {}; | |
| 739 | } | ||
| 740 | 12716 | ColumnStore& store = GetColumnStore(); | |
| 741 | 12716 | device::Queue& queue = store.GetDevice().GetQueue(); | |
| 742 | 12716 | if (Dense()) { | |
| 743 | 12707 | store.Flush(); | |
| 744 | 12707 | return queue.ReadAsync( | |
| 745 | 12707 | *GetTable().Contiguous(m_column.index, Rows().blocks.front().firstRow, Size())); | |
| 746 | } | ||
| 747 | 9 | const Ref<device::Buffer> cells = GatherCells(); | |
| 748 | 9 | return queue.ReadAsync(cells->Slice(0, uint64_t(CellBytes()) * Size())); | |
| 749 | 12716 | } | |
| 750 | |||
| 751 | 12658 | std::vector<std::byte> UntypedAttribute::ReadHost() const { | |
| 752 | 12658 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 753 | 12658 | CHRONO_BATCH_RENDER_ZONE("Scene/Read"); | |
| 754 | 12658 | const device::Download download = ReadHostAsync(); | |
| 755 | 12658 | download.Wait(); | |
| 756 | 12658 | const std::span<const std::byte> bytes = download.Bytes(); | |
| 757 | 25316 | return {bytes.begin(), bytes.end()}; | |
| 758 | 12658 | } | |
| 759 | |||
| 760 | 2 | device::Tensor UntypedAttribute::Read(Stream consumer) const { | |
| 761 | 2 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 762 | 2 | CHRONO_BATCH_RENDER_ZONE("Scene/Read"); | |
| 763 | 2 | const Ref<device::Buffer> cells = GatherCells(); | |
| 764 | 2 | GetColumnStore().GetDevice().GetQueue().Mark().Wait(consumer); | |
| 765 | 2 | return device::Tensor(cells, 0, Layout(m_column.column->type).type, | |
| 766 | 8 | Shape(Size(), Layout(m_column.column->type))); | |
| 767 | 2 | } | |
| 768 | |||
| 769 | 733 | void UntypedAttribute::SetArray(uint32_t index, Span<const std::byte> items) { | |
| 770 | 733 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 771 | 733 | CHRONO_BATCH_RENDER_ZONE("Scene/Write"); | |
| 772 | 733 | CheckWritable(true); | |
| 773 | 733 | CheckNotAttached(); | |
| 774 | 732 | if (!IsArray(m_column.column->type)) { | |
| 775 | ✗ | throw Error("{}'s {} holds no arrays", Info(m_column.schema).name, m_column.column->field); | |
| 776 | } | ||
| 777 | 732 | const uint32_t row = Rows().rows.at(index); | |
| 778 | 732 | if (row == kNoRow) { | |
| 779 | ✗ | throw Error("Prim {} of the view was removed", index); | |
| 780 | } | ||
| 781 | 732 | ColumnStore& store = GetColumnStore(); | |
| 782 | 732 | ArrayPool& arrays = store.Arrays(m_column.column->type); | |
| 783 | 732 | if (items.size() % arrays.ItemBytes() != 0) { | |
| 784 | ✗ | throw Error("{} bytes for items of {} bytes", items.size(), arrays.ItemBytes()); | |
| 785 | } | ||
| 786 | 732 | const uint32_t count = uint32_t(items.size() / arrays.ItemBytes()); | |
| 787 | 732 | const ArrayRef current = store.Array(m_column, row); | |
| 788 | 288 | const bool reused = current.address != 0 && current.count == count && | |
| 789 | 894 | arrays.References(current) == 1 && !arrays.Borrowed(current); | |
| 790 | 732 | const ArrayRef array = reused ? current : arrays.Allocate(count); | |
| 791 | 732 | arrays.Stage(array, items); | |
| 792 | 732 | store.SetArray(m_column, row, array); | |
| 793 | 732 | if (m_column.column->type == ColumnType::PrimArray) { | |
| 794 | ✗ | std::vector<uint32_t> targets(count); | |
| 795 | ✗ | std::memcpy(targets.data(), items.data(), items.size()); | |
| 796 | ✗ | m_view->GetStructure().MarkTargets(targets); | |
| 797 | ✗ | } | |
| 798 | 732 | store.StampWorld(m_view->GetStructure().GetPrims().Record(m_view->Prims()[index]).world); | |
| 799 | 734 | } | |
| 800 | |||
| 801 | 3973 | std::vector<std::byte> UntypedAttribute::GetArray(uint32_t index) const { | |
| 802 | 3973 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 803 | 3973 | CHRONO_BATCH_RENDER_ZONE("Scene/Read"); | |
| 804 | 3973 | const uint32_t row = Rows().rows.at(index); | |
| 805 | 3973 | if (row == kNoRow) { | |
| 806 | ✗ | throw Error("Prim {} of the view was removed", index); | |
| 807 | } | ||
| 808 | 3973 | ColumnStore& store = GetColumnStore(); | |
| 809 | 3973 | store.Flush(); | |
| 810 | 3973 | const ArrayRef array = store.Array(m_column, row); | |
| 811 | 3973 | return array.address == 0 ? std::vector<std::byte>{} | |
| 812 | 7946 | : store.Arrays(m_column.column->type).Read(array); | |
| 813 | 3973 | } | |
| 814 | |||
| 815 | 2281 | void UntypedAttribute::CheckArrayType(ColumnType type) const { | |
| 816 | 2281 | if (m_column.column->type != type) { | |
| 817 | 3 | throw Error("{}'s {} doesn't hold that kind of array", Info(m_column.schema).name, | |
| 818 | 2 | m_column.column->field); | |
| 819 | } | ||
| 820 | 2280 | } | |
| 821 | |||
| 822 | 299 | void UntypedAttribute::SetTargets(uint32_t index, const std::vector<std::string>& paths) { | |
| 823 | 299 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 824 | 299 | CheckWritable(true); | |
| 825 | 299 | CheckNotAttached(); | |
| 826 | 299 | CheckArrayType(ColumnType::PrimArray); | |
| 827 | 299 | if (Rows().rows.at(index) == kNoRow) { | |
| 828 | ✗ | throw Error("Prim {} of the view was removed", index); | |
| 829 | } | ||
| 830 | 299 | Structure& structure = m_view->GetStructure(); | |
| 831 | 299 | const uint32_t prim = m_view->Prims()[index]; | |
| 832 | 299 | structure.SetTargets(prim, Rows().rows.at(index), m_column, paths); | |
| 833 | 299 | GetColumnStore().StampWorld(structure.GetPrims().Record(prim).world); | |
| 834 | 299 | } | |
| 835 | |||
| 836 | 416 | std::vector<std::string> UntypedAttribute::GetTargets(uint32_t index) const { | |
| 837 | 416 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 838 | 416 | CheckArrayType(ColumnType::PrimArray); | |
| 839 | 416 | const Structure& structure = m_view->GetStructure(); | |
| 840 | 416 | std::vector<std::string> paths; | |
| 841 | 546 | for (const uint32_t prim : Items<uint32_t>(GetArray(index))) { | |
| 842 | 130 | if (prim != kNoPrim && structure.GetPrims().Alive(prim)) { | |
| 843 | 248 | paths.push_back(structure.GetPaths().Path(prim)); | |
| 844 | } | ||
| 845 | 416 | } | |
| 846 | 416 | return paths; | |
| 847 | 416 | } | |
| 848 | |||
| 849 | 530 | void UntypedAttribute::SetTokens(uint32_t index, const std::vector<std::string>& texts) { | |
| 850 | 530 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 851 | 530 | CheckArrayType(ColumnType::TokenArray); | |
| 852 | 529 | const std::vector<uint32_t> ids = InternTokens(m_view->GetStructure().GetTokens(), texts); | |
| 853 | 529 | SetArray(index, AsBytes(Span<const uint32_t>(ids))); | |
| 854 | 530 | } | |
| 855 | |||
| 856 | 1036 | std::vector<std::string> UntypedAttribute::GetTokens(uint32_t index) const { | |
| 857 | 1036 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 858 | 1036 | CheckArrayType(ColumnType::TokenArray); | |
| 859 | 1036 | const Tokens& tokens = m_view->GetStructure().GetTokens(); | |
| 860 | 1036 | std::vector<std::string> texts; | |
| 861 | 1271 | for (const uint32_t id : Items<uint32_t>(GetArray(index))) { | |
| 862 | 235 | texts.emplace_back(tokens.Text(Token{id})); | |
| 863 | 1036 | } | |
| 864 | 1036 | return texts; | |
| 865 | 1036 | } | |
| 866 | |||
| 867 | 14 | void UntypedAttribute::SetPrimvars(uint32_t index, const std::vector<Primvar>& primvars) { | |
| 868 | 14 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 869 | 14 | if (m_column.column->field != "primvars") { | |
| 870 | ✗ | throw Error("{}'s {} holds no primvars", Info(m_column.schema).name, | |
| 871 | ✗ | m_column.column->field); | |
| 872 | } | ||
| 873 | 14 | const std::vector<int32_t> table = PackPrimvars(m_view->GetStructure().GetTokens(), primvars); | |
| 874 | 14 | SetArray(index, AsBytes(Span<const int32_t>(table))); | |
| 875 | 14 | } | |
| 876 | |||
| 877 | 14 | std::vector<Primvar> UntypedAttribute::GetPrimvars(uint32_t index) const { | |
| 878 | 14 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 879 | 14 | if (m_column.column->field != "primvars") { | |
| 880 | ✗ | throw Error("{}'s {} holds no primvars", Info(m_column.schema).name, | |
| 881 | ✗ | m_column.column->field); | |
| 882 | } | ||
| 883 | 28 | return UnpackPrimvars(m_view->GetStructure().GetTokens(), Items<int32_t>(GetArray(index))); | |
| 884 | 14 | } | |
| 885 | |||
| 886 | 22 | void UntypedAttribute::MapArray(Mapping& mapping) { | |
| 887 | 22 | const ViewRows& rows = Rows(); | |
| 888 | 22 | ColumnStore& store = GetColumnStore(); | |
| 889 | 22 | store.Flush(); | |
| 890 | 22 | ArrayPool& heap = store.Arrays(m_column.column->type); | |
| 891 | 22 | const ColumnLayout layout = Layout(m_column.column->type); | |
| 892 | 22 | const bool writes = mapping.m_access != MapAccess::Read; | |
| 893 | 22 | if (writes && m_column.column->type == ColumnType::PrimArray) { | |
| 894 | ✗ | m_view->GetStructure().MarkUntrackedTargets(); | |
| 895 | } | ||
| 896 | 22 | std::vector<ArrayRef> arrays; | |
| 897 | 22 | std::vector<uint32_t> offsets = {0}; | |
| 898 | 68 | for (const uint32_t row : rows.rows) { | |
| 899 | 46 | if (row == kNoRow) { | |
| 900 | ✗ | throw Error("A mapping of arrays of removed prims"); | |
| 901 | } | ||
| 902 | 46 | arrays.push_back(store.Array(m_column, row)); | |
| 903 | 46 | offsets.push_back(offsets.back() + arrays.back().count); | |
| 904 | } | ||
| 905 | 22 | const uint64_t item = heap.ItemBytes(); | |
| 906 | 22 | const uint32_t count = arrays.empty() ? 0 : arrays.front().count; | |
| 907 | 22 | const bool equal = count > 0 && std::ranges::all_of(arrays, [&](const ArrayRef& array) { | |
| 908 | 43 | return array.count == count; | |
| 909 | 22 | }); | |
| 910 | 44 | const auto first = std::ranges::find_if(arrays, [](const ArrayRef& a) { return a.count > 0; }); | |
| 911 | 22 | uint64_t stride = uint64_t(count) * item; | |
| 912 | 22 | if (equal && arrays.size() > 1 && arrays[1].address > arrays[0].address) { | |
| 913 | 6 | stride = arrays[1].address - arrays[0].address; | |
| 914 | } | ||
| 915 | 22 | bool run = first != arrays.end() && stride % device::ElementBytes(layout.type) == 0; | |
| 916 | ✗ | if (run) { | |
| 917 | 22 | const device::BufferRange range = heap.Range(*first); | |
| 918 | 22 | const device::Buffer& buffer = range.GetBuffer(); | |
| 919 | 22 | uint64_t expected = first->address; | |
| 920 | 68 | for (const ArrayRef& array : arrays) { | |
| 921 | 46 | if (array.count == 0) { | |
| 922 | 3 | continue; | |
| 923 | } | ||
| 924 | 43 | run = run && array.address == expected && &heap.Range(array).GetBuffer() == &buffer && | |
| 925 | 25 | (!writes || heap.References(array) == 1); | |
| 926 | 43 | expected += equal ? stride : uint64_t(array.count) * item; | |
| 927 | } | ||
| 928 | } | ||
| 929 | |||
| 930 | 22 | device::Device& device = store.GetDevice(); | |
| 931 | 22 | device::Queue& queue = device.GetQueue(); | |
| 932 | 22 | Ref<device::Buffer> owner; | |
| 933 | 22 | uint64_t offset = 0; | |
| 934 | 22 | if (first == arrays.end()) { | |
| 935 | ✗ | owner = device.CreateBuffer({.bytes = 4, .label = "empty array mapping"}); | |
| 936 | 22 | } else if (run) { | |
| 937 | 15 | const device::BufferRange range = heap.Range(*first); | |
| 938 | 15 | owner = Ref<device::Buffer>(&range.GetBuffer()); | |
| 939 | 15 | offset = range.Offset(); | |
| 940 | } else { | ||
| 941 | 7 | std::vector<uint32_t> counts(arrays.size()); | |
| 942 | 27 | for (uint32_t i = 0; i < arrays.size(); ++i) { | |
| 943 | 20 | counts[i] = arrays[i].count; | |
| 944 | } | ||
| 945 | 7 | std::vector<ArrayRef> fresh; | |
| 946 | 7 | if (writes) { | |
| 947 | 10 | fresh = heap.AllocateRun(counts); | |
| 948 | } else { | ||
| 949 | 2 | owner = device.CreateBuffer({.bytes = uint64_t(offsets.back()) * item, | |
| 950 | .usage = device::BufferUsage::Interop, | ||
| 951 | 4 | .label = "array mapping staging"}); | |
| 952 | } | ||
| 953 | 7 | device::CommandEncoder encoder = queue.CreateEncoder(); | |
| 954 | 27 | for (uint32_t i = 0; i < arrays.size(); ++i) { | |
| 955 | 20 | if (arrays[i].count == 0) { | |
| 956 | 2 | continue; | |
| 957 | } | ||
| 958 | 18 | const uint64_t bytes = uint64_t(arrays[i].count) * item; | |
| 959 | 36 | encoder.Copy(heap.Range(arrays[i]).Slice(0, bytes), | |
| 960 | 31 | writes ? heap.Range(fresh[i]).Slice(0, bytes) | |
| 961 | 5 | : owner->Slice(uint64_t(offsets[i]) * item, bytes)); | |
| 962 | } | ||
| 963 | 7 | queue.Submit(encoder.Finish()); | |
| 964 | 7 | if (writes) { | |
| 965 | 19 | for (uint32_t i = 0; i < arrays.size(); ++i) { | |
| 966 | 14 | if (arrays[i].count > 0) { | |
| 967 | 13 | store.SetArray(m_column, rows.rows[i], fresh[i]); | |
| 968 | } | ||
| 969 | } | ||
| 970 | 5 | store.Flush(); | |
| 971 | 5 | const device::BufferRange range = heap.Range(fresh[first - arrays.begin()]); | |
| 972 | 5 | owner = Ref<device::Buffer>(&range.GetBuffer()); | |
| 973 | 5 | offset = range.Offset(); | |
| 974 | } | ||
| 975 | 7 | stride = uint64_t(count) * item; | |
| 976 | 7 | } | |
| 977 | |||
| 978 | 22 | const uint64_t element = device::ElementBytes(layout.type); | |
| 979 | 22 | std::vector<int64_t> shape; | |
| 980 | 22 | std::vector<int64_t> strides; | |
| 981 | 22 | if (equal) { | |
| 982 | 18 | shape = {int64_t(arrays.size()), int64_t(count)}; | |
| 983 | 18 | strides = {int64_t(stride / element), int64_t(layout.lanes)}; | |
| 984 | } else { | ||
| 985 | 4 | shape = {int64_t(offsets.back())}; | |
| 986 | 4 | strides = {int64_t(layout.lanes)}; | |
| 987 | 12 | mapping.m_offsets = device::Tensor(device.CreateBuffer({.label = "array mapping offsets"}, | |
| 988 | std::span<const uint32_t>(offsets)), | ||
| 989 | 8 | 0, device::DType::UInt32, {int64_t(offsets.size())}); | |
| 990 | } | ||
| 991 | 22 | if (layout.lanes > 1) { | |
| 992 | 8 | shape.push_back(layout.lanes); | |
| 993 | 8 | strides.push_back(1); | |
| 994 | } | ||
| 995 | 22 | mapping.m_values = | |
| 996 | 22 | device::Tensor(owner, offset, layout.type, std::move(shape), std::move(strides)); | |
| 997 | 22 | mapping.m_aliased = run || writes; | |
| 998 | 22 | mapping.m_ready = queue.Mark(); | |
| 999 | 56 | } | |
| 1000 | |||
| 1001 | 35 | Mapping UntypedAttribute::Map(MapAccess access, Stream consumer) { | |
| 1002 | 35 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 1003 | 35 | if (access != MapAccess::Read) { | |
| 1004 | 28 | CheckWritable(true); | |
| 1005 | 28 | CheckHostWritten(m_column); | |
| 1006 | 28 | const Attachment* attachment = GetColumnStore().AttachmentCount() > 0 | |
| 1007 | 28 | ? GetColumnStore().FindAttachment(m_column, Rows().rows) | |
| 1008 | 28 | : nullptr; | |
| 1009 | 28 | if (attachment == nullptr) { | |
| 1010 | 25 | CheckNotAttached(); | |
| 1011 | 3 | } else if (!attachment->inPlace || IsArray(m_column.column->type)) { | |
| 1012 | 3 | throw Error("{}'s {} is attached as arrays or by copy, so it doesn't map for writing; " | |
| 1013 | "write the attached tensor and touch the attribute", | ||
| 1014 | 6 | Info(m_column.schema).name, m_column.column->field); | |
| 1015 | } | ||
| 1016 | } | ||
| 1017 | 32 | ColumnStore& store = GetColumnStore(); | |
| 1018 | 32 | store.Flush(); | |
| 1019 | 32 | const ColumnLayout layout = Layout(m_column.column->type); | |
| 1020 | 32 | device::Queue& queue = store.GetDevice().GetQueue(); | |
| 1021 | 32 | Mapping mapping; | |
| 1022 | 32 | mapping.m_view = m_view; | |
| 1023 | 32 | mapping.m_column = m_column; | |
| 1024 | 32 | mapping.m_access = access; | |
| 1025 | 32 | if (IsArray(m_column.column->type)) { | |
| 1026 | 22 | MapArray(mapping); | |
| 1027 | 10 | } else if (Size() > 0 && Dense()) { | |
| 1028 | 4 | const device::BufferRange range = | |
| 1029 | 4 | *GetTable().Contiguous(m_column.index, Rows().blocks.front().firstRow, Size()); | |
| 1030 | 8 | mapping.m_values = device::Tensor(Ref<device::Buffer>(&range.GetBuffer()), range.Offset(), | |
| 1031 | 12 | layout.type, Shape(Size(), layout)); | |
| 1032 | 4 | mapping.m_aliased = true; | |
| 1033 | 4 | mapping.m_ready = queue.Mark(); | |
| 1034 | 6 | } else if (Size() > 0) { | |
| 1035 | 6 | Ref<device::Buffer> staging; | |
| 1036 | 6 | if (access == MapAccess::Write) { | |
| 1037 | 2 | staging = store.GetDevice().CreateBuffer({.bytes = uint64_t(CellBytes()) * Size(), | |
| 1038 | .usage = device::BufferUsage::Interop, | ||
| 1039 | 2 | .label = "mapping staging"}); | |
| 1040 | } else { | ||
| 1041 | 5 | staging = GatherCells(); | |
| 1042 | } | ||
| 1043 | 6 | mapping.m_values = device::Tensor(staging, 0, layout.type, Shape(Size(), layout)); | |
| 1044 | 12 | mapping.m_rows = device::Tensor(m_view->DeviceRows(m_column.schema), 0, | |
| 1045 | 12 | device::DType::UInt32, {int64_t(Size())}); | |
| 1046 | 6 | mapping.m_ready = queue.Mark(); | |
| 1047 | 6 | } | |
| 1048 | 32 | if (consumer.IsCompute()) { | |
| 1049 | ✗ | mapping.m_ready.Wait(consumer); | |
| 1050 | } | ||
| 1051 | 32 | store.Pin(m_column.schema); | |
| 1052 | 32 | mapping.m_open = true; | |
| 1053 | 32 | return mapping; | |
| 1054 | 38 | } | |
| 1055 | |||
| 1056 | 15 | Mapping UntypedAttribute::MapPrimvar(std::string_view name, MapAccess access, Stream consumer) { | |
| 1057 | 15 | const auto lock = LockStore(m_view ? m_view->GetColumnStore().Lock() : nullptr); | |
| 1058 | 15 | if (m_column.column->field != "primvars") { | |
| 1059 | 3 | throw Error("{}'s {} holds no primvars", Info(m_column.schema).name, | |
| 1060 | 2 | m_column.column->field); | |
| 1061 | } | ||
| 1062 | 14 | Mapping mapping = Map(access); | |
| 1063 | 14 | const device::Tensor tables = mapping.Values(); | |
| 1064 | 14 | if (tables.Shape().size() != 2) { | |
| 1065 | 1 | mapping.Close(); | |
| 1066 | 1 | throw Error("The primvars of the mapped prims differ in size, so {} doesn't map as one " | |
| 1067 | "tensor", | ||
| 1068 | 1 | name); | |
| 1069 | } | ||
| 1070 | 13 | ColumnStore& store = GetColumnStore(); | |
| 1071 | 13 | std::vector<ArrayRef> arrays; | |
| 1072 | 38 | for (const uint32_t row : Rows().rows) { | |
| 1073 | 50 | arrays.push_back(store.Array(m_column, row)); | |
| 1074 | } | ||
| 1075 | 13 | PrimvarLayout& layout = m_view->Layout(m_column, name); | |
| 1076 | 13 | const bool cached = | |
| 1077 | 15 | layout.tables.size() == arrays.size() && | |
| 1078 | 2 | std::ranges::equal(layout.tables, arrays, [](const ArrayRef& a, const ArrayRef& b) { | |
| 1079 | 5 | return a.address == b.address && a.count == b.count; | |
| 1080 | 21 | }); | |
| 1081 | 11 | if (!cached) { | |
| 1082 | 11 | const uint32_t words = arrays.front().count; | |
| 1083 | 11 | const uint32_t stride = uint32_t(tables.Strides()[0]); | |
| 1084 | 11 | device::Queue& queue = store.GetDevice().GetQueue(); | |
| 1085 | 11 | uint32_t count = 0; | |
| 1086 | 11 | queue.Read(tables.GetBuffer().Slice(tables.Offset(), sizeof(uint32_t)), &count); | |
| 1087 | 11 | const uint32_t header = std::min(words, 1 + count * kPrimvarEntryWords); | |
| 1088 | 11 | const Ref<device::Buffer> staging = store.GetDevice().CreateBuffer( | |
| 1089 | 11 | {.bytes = uint64_t(header) * sizeof(uint32_t) * arrays.size(), | |
| 1090 | 11 | .label = "primvar headers"}); | |
| 1091 | 11 | device::CommandEncoder encoder = queue.CreateEncoder(); | |
| 1092 | 31 | for (uint32_t i = 0; i < arrays.size(); ++i) { | |
| 1093 | 40 | encoder.Copy( | |
| 1094 | 20 | tables.GetBuffer().Slice(tables.Offset() + uint64_t(i) * stride * sizeof(uint32_t), | |
| 1095 | uint64_t(header) * sizeof(uint32_t)), | ||
| 1096 | 20 | staging->Slice(uint64_t(i) * header * sizeof(uint32_t), | |
| 1097 | uint64_t(header) * sizeof(uint32_t))); | ||
| 1098 | } | ||
| 1099 | 11 | queue.Submit(encoder.Finish()); | |
| 1100 | 14 | std::vector<uint32_t> headers(size_t(header) * arrays.size()); | |
| 1101 | 11 | queue.Read(staging->Slice(0), headers.data()); | |
| 1102 | 19 | for (uint32_t i = 1; i < arrays.size(); ++i) { | |
| 1103 | 9 | if (!std::equal(headers.begin(), headers.begin() + header, | |
| 1104 | 9 | headers.begin() + size_t(i) * header)) { | |
| 1105 | 1 | mapping.Close(); | |
| 1106 | 1 | throw Error("The mapped prims lay out their primvars differently, so {} doesn't " | |
| 1107 | "map as one tensor", | ||
| 1108 | 1 | name); | |
| 1109 | } | ||
| 1110 | } | ||
| 1111 | 10 | const uint32_t token = m_view->GetStructure().GetTokens().Find(name).id; | |
| 1112 | 10 | bool found = false; | |
| 1113 | 21 | for (uint32_t slot = 0; slot < count && token != 0; ++slot) { | |
| 1114 | 11 | PrimvarEntry entry; | |
| 1115 | 11 | std::memcpy(&entry, headers.data() + 1 + slot * kPrimvarEntryWords, sizeof(entry)); | |
| 1116 | 11 | if (entry.name == token) { | |
| 1117 | 8 | layout.entry = entry; | |
| 1118 | 8 | found = true; | |
| 1119 | } | ||
| 1120 | } | ||
| 1121 | 10 | if (!found) { | |
| 1122 | 2 | mapping.Close(); | |
| 1123 | 4 | throw Error("{} has no primvar {}", Info(m_column.schema).name, name); | |
| 1124 | } | ||
| 1125 | 8 | layout.tables = arrays; | |
| 1126 | 33 | } | |
| 1127 | 10 | const PrimvarEntry& entry = layout.entry; | |
| 1128 | 10 | const auto lanes = int64_t(PrimvarComponents(PrimvarType(entry.type)) * entry.elementSize); | |
| 1129 | 10 | std::vector<int64_t> shape = {int64_t(arrays.size()), int64_t(entry.valueCount)}; | |
| 1130 | 10 | std::vector<int64_t> strides = {tables.Strides()[0], lanes}; | |
| 1131 | 10 | if (lanes > 1) { | |
| 1132 | 3 | shape.push_back(lanes); | |
| 1133 | 3 | strides.push_back(1); | |
| 1134 | } | ||
| 1135 | 40 | mapping.m_values = device::Tensor( | |
| 1136 | 20 | Ref<device::Buffer>(&tables.GetBuffer()), | |
| 1137 | 10 | tables.Offset() + uint64_t(entry.values) * sizeof(uint32_t), | |
| 1138 | 10 | PrimvarIsInt(PrimvarType(entry.type)) ? device::DType::Int32 : device::DType::Float32, | |
| 1139 | 10 | std::move(shape), std::move(strides)); | |
| 1140 | 10 | if (consumer.IsCompute()) { | |
| 1141 | ✗ | mapping.m_ready.Wait(consumer); | |
| 1142 | } | ||
| 1143 | 10 | return mapping; | |
| 1144 | 48 | } | |
| 1145 | |||
| 1146 | } | ||
| 1147 |