Decoding camera outputs#
Example USD#
A RenderProduct that outputs both LdrColor and HdrColor from a camera:
#usda 1.0
def "Render" {
def RenderProduct "Camera" {
int2 resolution = (1920, 1080)
rel camera = </World/Camera>
rel orderedVars = [<LdrColor>, <HdrColor>]
def RenderVar "LdrColor" {
string sourceName = "LdrColor"
}
def RenderVar "HdrColor" {
string sourceName = "HdrColor"
}
}
}
from pxr import Gf, UsdRender
render_scope = stage.DefinePrim("/Render")
camera_product = UsdRender.Product.Define(stage, "/Render/Camera")
camera_product.GetResolutionAttr().Set(Gf.Vec2i(1920, 1080))
camera_product.GetCameraRel().SetTargets(["/World/Camera"])
ldr_var = UsdRender.Var.Define(stage, "/Render/Camera/LdrColor")
ldr_var.GetSourceNameAttr().Set("LdrColor")
hdr_var = UsdRender.Var.Define(stage, "/Render/Camera/HdrColor")
hdr_var.GetSourceNameAttr().Set("HdrColor")
camera_product.GetOrderedVarsRel().SetTargets([
"/Render/Camera/LdrColor",
"/Render/Camera/HdrColor",
])
Accessing Outputs in Code#
After stepping the renderer with a RenderProduct path, the outputs are available by full RenderVar prim path:
import numpy as np
from PIL import Image
import ovrtx
products = renderer.step(
render_products={"/Render/Camera"},
delta_time=1.0 / 60,
ordinal=ordinal,
)
for frame in products["/Render/Camera"].frames:
# RenderVar outputs are keyed by their full prim paths.
ldr_var = frame.render_vars["/Render/Camera/LdrColor"]
hdr_var = frame.render_vars["/Render/Camera/HdrColor"]
# LdrColor: uint8 sRGB image
var = ldr_var.map(device=ovrtx.Device.CPU)
ldr_pixels = np.from_dlpack(var) # shape: (H, W, 4), dtype: uint8
assert ldr_pixels.shape == (1080, 1920, 4)
assert ldr_pixels.dtype == np.uint8
Image.fromarray(ldr_pixels).save(output_dir / "test_camera_sensors.LdrColor.png")
# HdrColor: float16 linear image
var = hdr_var.map(device=ovrtx.Device.CPU)
hdr_pixels = np.from_dlpack(var) # shape: (H, W, 4), dtype: float16
assert hdr_pixels.shape == (1080, 1920, 4)
assert hdr_pixels.dtype == np.float16
ovrtx_render_var_output_handle_t ldr_handle{};
ovrtx_render_var_output_handle_t hdr_handle{};
// This step requested one RenderProduct. Select its first output frame,
// then find RenderVars by their full prim paths.
ovrtx_render_product_frame_output_t const& frame = outputs.outputs[0].output_frames[0];
for (size_t var_index = 0; var_index < frame.render_var_count; ++var_index) {
ovrtx_render_product_render_var_output_t const& var = frame.output_render_vars[var_index];
if (!var.render_var_path.ptr) {
continue;
}
std::string_view const path(var.render_var_path.ptr, var.render_var_path.length);
if (path == "/Render/Camera/LdrColor") {
ldr_handle = var.output_handle;
} else if (path == "/Render/Camera/HdrColor") {
hdr_handle = var.output_handle;
}
}
ASSERT_NE(ldr_handle, OVRTX_INVALID_RENDER_VAR_OUTPUT_HANDLE);
ASSERT_NE(hdr_handle, OVRTX_INVALID_RENDER_VAR_OUTPUT_HANDLE);
// Map LdrColor to CPU
ovrtx_map_output_description_t map_desc = {};
map_desc.device_type = OVRTX_MAP_DEVICE_TYPE_CPU;
ovrtx_render_var_output_t ldr_output = {};
result = ovrtx_map_render_var_output(renderer_, ldr_handle, &map_desc,
ovrtx_timeout_infinite, &ldr_output);
ASSERT_API_SUCCESS(result.status);
DLTensor const& ldr_tensor = *ldr_output.tensors[0].dl;
// ldr_tensor.data: RGBA uint8 pixels
// ldr_tensor.shape[0]: height, ldr_tensor.shape[1]: width
save_ldr_png("CameraSensors.LdrColor",
ldr_tensor.data,
static_cast<int>(ldr_tensor.shape[1]),
static_cast<int>(ldr_tensor.shape[0]));
ovrtx_cuda_sync_t no_sync = {};
ovrtx_unmap_render_var_output(renderer_, ldr_output.map_handle, no_sync);
Image tensors#
Applies to NormalSD, DepthSD, Depth, XRDepth, DistanceToCameraSD, DistanceToImagePlaneSD, DiffuseAlbedoSD, Camera3dPositionSD, TargetMotionSD, SemanticSegmentation, SemanticInstanceSegmentation, StableIdSegmentation, and NonStableInstanceSegmentation.
Image tensors already carry their scalar type and shape in DLPack. Python preserves
that type; C callers inspect dtype.code, dtype.bits, and shape before
reading pixels. The C helper checks CPU placement and contiguous layout and adds
byte_offset. Keep mappings alive while using their pointers.
Float images use float32; albedo uses uint8. Segmentation images use uint32: keep all four words for stable and semantic IDs. Color IDs only for display; do not normalize them before joining records. Non-stable IDs are runtime indices. Map normals from [-1, 1] to display colors, normalize finite distances for display, and encode motion direction as hue and displacement magnitude as brightness. The table gives each output’s units and channels. See Identifier domains and stability before joining segmentation pixels to record outputs.
def map_sd_image(frame, render_var_path: str) -> np.ndarray:
"""Copy a typed image to CPU memory before its mapping is released."""
mapped = frame.render_vars[render_var_path].map(device=ovrtx.Device.CPU)
return np.from_dlpack(mapped).copy()
// The output must already be mapped with OVRTX_MAP_DEVICE_TYPE_CPU.
// Return the first pixel only for contiguous, channel-last image tensors.
static const void* sd_image_data(const DLTensor* tensor) {
if (!tensor || tensor->device.device_type != kDLCPU || tensor->ndim != 3 ||
tensor->dtype.lanes != 1 || !tensor->data || !tensor->shape) return nullptr;
if (tensor->strides && (tensor->strides[2] != 1 ||
tensor->strides[1] != tensor->shape[2] ||
tensor->strides[0] != tensor->shape[1] * tensor->shape[2])) return nullptr;
return static_cast<const uint8_t*>(tensor->data) + tensor->byte_offset;
}
Identifier domains and stability#
These outputs use several identifier domains. Join outputs from the same scene snapshot and compare every word of a four-word ID. A row number, runtime instance ID, semantic class ID, and stable ID are different keys even when their numeric values happen to match.
Key |
Where it appears and how to join it |
Lifetime and consistency |
|---|---|---|
Runtime instance ID (uint32) |
|
Assigned to renderer instances. Scene edits, rebuilding instances, or reloading a stage can change or reuse these values. They are not persistent prim IDs and have no run-to-run or cross-machine guarantee. |
Semantic-instance row index (uint32) |
|
An index into the current semantic data. Preserve sparse rows; do not cache the index across scene edits or assume it identifies a class. |
Non-stable semantic-instance pixel (uint32) |
|
Has the same lifetime as the semantic-instance row index. It cannot be
compared directly with a raw box’s |
Geometry stable ID (four uint32 words) |
|
Derived from instance identity, including the canonical instance path and subset identity. Motion alone does not change that identity; renaming, reparenting, or changing instancing/subsets can change it. |
Semantic-instance stable ID (four uint32 words) |
|
Derived from the semantic instance’s identity. Inheritance, point-instance identity and subset identity matter; the visible leaf prim path alone does not determine it. |
Semantic class ID |
|
Allocated for compound label sets in the current runtime. Equal label sets share a class ID while that registry is retained. Labels, filters, initialization order or stage resets can change the mapping; the same label is not guaranteed the same number across runs or machines. |
Prim token (uint64) |
|
Runtime metadata, not a portable path string or a persistent identifier. |
For example, to connect a raw box to a label, use its instance_id to look up
the semantic-instance row through Instance map. Read that row’s
semantic_id from Stable-to-semantic pairs, then find the matching
four-word id in Identifier maps and deltas. Use the row’s stable_id to
join semantic boxes or semantic-instance pixels. This also handles a labelled
parent that groups several rendered children.
Stable means derived from scene identity rather than allocation order.
Repeated runs with the same identity inputs and ID-generation implementation
can reproduce these IDs, but the SDK does not promise a portable hash format
across versions, rendering backends, or machines. A matching visible prim path
alone is insufficient, especially for instancing and subsets. For persistent
datasets, save the accompanying path and label maps with each capture, and use
application-owned identifiers when you require a cross-run or cross-machine
identity contract. Always consume delta maps from the first frame; an empty
later delta is not a replacement for the full map.
Legacy byte buffers#
The structured outputs documented below expose named fields and do not require
private record layouts. For a remaining legacy one-dimensional uint8 buffer,
B in shape (B,) is its byte length. Use its documented record type before
applying these generic helpers; do not normalize metadata as image pixels.
The helpers validate byte lengths. In C, memcpy() avoids alignment assumptions.
counted=true reads a final uint32 count only for formats that specify one.
def decode_sd_records(data: np.ndarray, dtype: np.dtype, counted: bool = False) -> np.ndarray:
"""Decode little-endian records, checking lengths before reading reserved buffers."""
buffer = np.ascontiguousarray(data).view(np.uint8).reshape(-1)
if counted:
# Counted legacy buffers end with their actual record count.
if buffer.size < 4:
raise ValueError("Missing record count")
count = int.from_bytes(buffer[-4:].tobytes(), "little")
if count > (buffer.size - 4) // dtype.itemsize:
raise ValueError("Record count exceeds buffer capacity")
else:
if buffer.size % dtype.itemsize:
raise ValueError("Incomplete record")
count = buffer.size // dtype.itemsize
return buffer[:count * dtype.itemsize].view(dtype)
typedef struct SdRecordView {
const uint8_t* data;
size_t count;
size_t stride;
} SdRecordView;
// Read little-endian buffer lengths before copying records with memcpy().
// Set counted only when a legacy record buffer ends with an explicit record count.
static bool sd_records(const uint8_t* data, size_t bytes, size_t stride,
bool counted, SdRecordView* output) {
if (!data || !stride || !output) return false;
size_t count = 0;
if (counted) {
if (bytes < 4) return false;
uint32_t stored_count = 0;
memcpy(&stored_count, data + bytes - 4, 4);
count = stored_count;
if (count > (bytes - 4) / stride) return false;
} else {
if (bytes % stride) return false;
count = bytes / stride;
}
*output = {data, count, stride};
return true;
}
Bounding boxes#
Applies to BoundingBox2DTightSD, BoundingBox2DLooseSD, BoundingBox3DSD, SemanticBoundingBox2DTight, SemanticBoundingBox2DLoose, and SemanticBoundingBox3D.
Bounding-box outputs use schema version 0 and expose typed named tensors.
Inspect names, DLPack types, shapes, strides and byte offsets; fields may be
non-contiguous views of one allocation. Semantic boxes reserve capacity: consume
only valid_count[0] rows. The count is on the requested device, so CUDA
consumers can use it without a host readback. Synchronize CUDA access with the
output and keep the mapping alive while using its fields.
Use semantic=True for semantic boxes and three_d=True for 3D boxes in Python.
In C, look up extent and read elements with their DLPack strides.
Use sd_box_count() to select valid semantic rows. Skip records whose maximum is below their minimum.
2D coordinates are pixel bounds; draw rectangles over the color image. Raw boxes
join visibility and token records by instance/tile ID. Semantic boxes carry full
128-bit instance and class IDs.
Each 3D bounding-box output contains local extent bounds and an
object_to_world tensor with one 4×4 matrix per row from local space to world space. The decoders below name this field
local_to_world. Use the matrix from the same record to transform the eight
box corners before projecting into the camera.
def decode_sd_boxes(data: dict, semantic: bool = False, three_d: bool = False) -> np.ndarray:
"""Read strided named fields; semantic valid_count limits capacity-sized tensors."""
extent = data["extent"]
dimensions = 3 if three_d else 2
if extent.ndim != 3 or extent.shape[1:] != (2, dimensions):
raise ValueError("Invalid bounding-box extent shape")
capacity = len(extent)
count = int(data["valid_count"][0]) if semantic else capacity
if not 0 <= count <= capacity:
raise ValueError("Bounding-box count exceeds capacity")
fields = {"stable_id": data["id"], "semantic_id": data["semantic_id"]} if semantic else {
"instance_id": data["instance_id"], "tile_id": data["render_tile_id"]}
fields.update(min=extent[:, 0], max=extent[:, 1])
if three_d:
fields["local_to_world"] = data["object_to_world"]
if any(len(value) != capacity for value in fields.values()):
raise ValueError("Bounding-box fields have unequal capacity")
dtype = np.dtype([(name, values.dtype, values.shape[1:]) for name, values in fields.items()])
result = np.empty(count, dtype=dtype)
for name, values in fields.items():
result[name] = values[:count]
return result
// Look up public fields by name, never by an assumed tensor slot or packed ABI.
static const DLTensor* sd_field(const ovrtx_render_var_output_t* output, const char* name, bool param = false) {
if (!output) return nullptr;
for (size_t i = 0; i < (param ? output->num_params : output->num_tensors); ++i) {
const ovx_string_t* key = param ? &output->params[i].name : output->tensors[i].name;
if (key && key->ptr && key->length == strlen(name) && !memcmp(key->ptr, name, key->length))
return param ? &output->params[i].dl : output->tensors[i].dl;
}
return nullptr;
}
// Read scalar elements in logical row-major order. DLPack strides are in elements;
// byte_offset is in bytes. CPU mappings must have completed before reading.
static bool sd_read(const DLTensor* tensor, size_t first, size_t count, uint8_t code, uint8_t bits, void* destination) {
if (!tensor || !destination || tensor->device.device_type != kDLCPU || tensor->dtype.code != code ||
tensor->dtype.bits != bits || tensor->dtype.lanes != 1 || !bits || bits % 8 || tensor->ndim < 0 ||
(tensor->ndim && !tensor->shape)) return false;
size_t size = 1;
for (int dim = 0; dim < tensor->ndim; ++dim) {
if (tensor->shape[dim] < 0 || (tensor->shape[dim] && size > SIZE_MAX / size_t(tensor->shape[dim])))
return false;
size *= size_t(tensor->shape[dim]);
}
if (first > size || count > size - first || (count && !tensor->data)) return false;
for (size_t element = 0; element < count; ++element) {
size_t index = first + element, offset = 0, contiguous = 1;
for (int dim = tensor->ndim; dim-- > 0;) {
if (tensor->strides && tensor->strides[dim] < 0) return false;
const size_t stride = tensor->strides ? size_t(tensor->strides[dim]) : contiguous;
offset += index % size_t(tensor->shape[dim]) * stride;
index /= size_t(tensor->shape[dim]);
contiguous *= size_t(tensor->shape[dim]);
}
memcpy(static_cast<uint8_t*>(destination) + element * (bits / 8),
static_cast<const uint8_t*>(tensor->data) + tensor->byte_offset + offset * (bits / 8), bits / 8);
}
return true;
}
// Raw boxes use extent.shape[0] rows. Semantic boxes have capacity-sized fields:
// consume only valid_count[0] rows, including when valid_count is zero.
static bool sd_box_count(const ovrtx_render_var_output_t* output, bool semantic, size_t* count) {
const DLTensor* extent = sd_field(output, "extent");
if (!extent || !extent->shape || extent->ndim != 3 || extent->shape[0] < 0 || !count) return false;
*count = size_t(extent->shape[0]);
if (semantic) {
uint32_t valid = 0;
if (!sd_read(sd_field(output, "valid_count"), 0, 1, kDLUInt, 32, &valid) || valid > *count) return false;
*count = valid;
}
return true;
}
// extent is int32[N,2,2] for 2D, float32[N,2,3] for 3D (minimum then maximum).
// Raw keys: instance_id and render_tile_id. Semantic keys: id[N,4], semantic_id[N,4].
// 3D boxes additionally publish object_to_world[N,4,4]. Use sd_read() for each field.
// Skip inverted bounds and clip 3D edges before projection.
Occlusion#
Applies to OcclusionSD.
Schema Occlusion version 0 exposes instance_id and render_tile_id as
uint32 tensors and occlusion_ratio as float32, all shape (N,). Fields may
be strided views of one backing allocation. Explicit CUDA mappings return GPU
tensors; default follows producer placement (normally GPU). CPU mappings read
back GPU data. CUDA-array mapping is unsupported. Join each instance/tile key to its raw bounding box,
then annotate that box with the ratio. Skip negative ratios;
valid ratios range from zero (unoccluded) to one (fully occluded).
def decode_sd_occlusion(data: dict) -> np.ndarray:
"""Read named occlusion fields; negative ratios are invalid."""
fields = {"instance_id": data["instance_id"], "tile_id": data["render_tile_id"],
"ratio": data["occlusion_ratio"]}
count = len(fields["instance_id"])
if any(value.shape != (count,) for value in fields.values()):
raise ValueError("Occlusion field shapes differ")
result = np.empty(count, dtype=[("instance_id", np.uint32), ("tile_id", np.uint32), ("ratio", np.float32)])
for name, value in fields.items():
result[name] = value
return result
// Application convenience type; source fields may have different element strides.
typedef struct SdOcclusion {
uint32_t instance_id, tile_id;
float ratio;
} SdOcclusion;
static bool sd_occlusion(const ovrtx_render_var_output_t* output, size_t row, SdOcclusion* value) {
return value && sd_read(sd_field(output, "instance_id"), row, 1, kDLUInt, 32, &value->instance_id) &&
sd_read(sd_field(output, "render_tile_id"), row, 1, kDLUInt, 32, &value->tile_id) &&
sd_read(sd_field(output, "occlusion_ratio"), row, 1, kDLFloat, 32, &value->ratio);
}
// Join to raw boxes by instance_id AND tile_id; negative ratios are invalid.
Instance token map#
Applies to InstanceIdTokenMapSD.
Schema InstanceIdTokenMap version 0 exposes instance_id, prototype_index,
and instance_index as uint32 tensors, and prim_token as uint64. Each has
shape (N,); there is no header row. CPU uint32 scalar params
min_instance_id, max_instance_id, and render_tile_count describe the
instance range and tile count; each param has shape (1,). A zero-row map
may have a minimum greater than its maximum. The token is an opaque runtime
handle, not a string address or a persistent network ID.
Default and CPU mappings return CPU tensors; explicit CUDA mappings return CUDA tensors while params stay on CPU. CUDA-array mapping is unsupported. Honor the returned CUDA synchronization and keep the mapping alive while consuming fields.
def decode_sd_tokens(data: dict) -> tuple[dict, np.ndarray]:
"""Read named instance fields and CPU scalar parameters; tensors contain no header row."""
fields = {"instance_id": data["instance_id"], "prim_token": data["prim_token"],
"prototype": data["prototype_index"], "instance_index": data["instance_index"]}
count = len(fields["instance_id"])
if any(value.shape != (count,) for value in fields.values()):
raise ValueError("Instance-token field shapes differ")
records = np.empty(count, dtype=[("instance_id", np.uint32), ("prim_token", np.uint64),
("prototype", np.uint32), ("instance_index", np.uint32)])
for name, value in fields.items():
records[name] = value
return data["params"], records
typedef struct SdInstanceToken {
uint32_t instance_id;
uint64_t prim_token;
uint32_t prototype, instance_index;
} SdInstanceToken;
typedef struct SdTokenParams {
uint32_t min_instance_id, max_instance_id, render_tile_count;
} SdTokenParams;
static bool sd_token(const ovrtx_render_var_output_t* output, size_t row,
SdInstanceToken* value, SdTokenParams* params) {
return value && params &&
sd_read(sd_field(output, "instance_id"), row, 1, kDLUInt, 32, &value->instance_id) &&
sd_read(sd_field(output, "prim_token"), row, 1, kDLUInt, 64, &value->prim_token) &&
sd_read(sd_field(output, "prototype_index"), row, 1, kDLUInt, 32, &value->prototype) &&
sd_read(sd_field(output, "instance_index"), row, 1, kDLUInt, 32, &value->instance_index) &&
sd_read(sd_field(output, "min_instance_id", true), 0, 1, kDLUInt, 32, ¶ms->min_instance_id) &&
sd_read(sd_field(output, "max_instance_id", true), 0, 1, kDLUInt, 32, ¶ms->max_instance_id) &&
sd_read(sd_field(output, "render_tile_count", true), 0, 1, kDLUInt, 32, ¶ms->render_tile_count);
}
// Tensor rows contain only instances. Params are CPU scalars even for CUDA maps.
// prim_token is an opaque runtime handle, not an address to a string.
Instance map#
Applies to InstanceMap.
Schema InstanceMap version 0 exposes one named uint32 tensor,
semantic_instance_index with shape (N,). Element i corresponds to
min_instance_id + i from the token-map CPU params; request
InstanceIdTokenMapSD alongside this output to obtain that origin. 0xFFFFFFFF means no
semantic parent. These values index semantic-instance rows, not semantic label
IDs or the four-word stable IDs. Default/CPU mappings return CPU tensors;
explicit CUDA mappings return CUDA tensors. CUDA-array mapping is unsupported.
def decode_sd_instance_map(data: dict) -> np.ndarray:
"""Element i belongs to min_instance_id + i from the token-map CPU params."""
return data["semantic_instance_index"]
static bool sd_instance_map(const ovrtx_render_var_output_t* output, size_t row, uint32_t* index) {
return index && sd_read(sd_field(output, "semantic_instance_index"), row, 1, kDLUInt, 32, index);
}
// Element i belongs to min_instance_id + i from the token-map CPU params.
// UINT32_MAX means no semantic parent; these indices are not semantic label IDs.
Identifier maps and deltas#
Applies to SemanticIdMap, SemanticIdMapDeltas, StableIdMap, and StableIdMapDeltas.
These outputs have type IdentifierMap, version 0, and three named tensors:
Tensor |
Dtype and shape |
Contents |
|---|---|---|
|
uint32 |
All four words of each identifier, in producer row order. |
|
uint64 |
Offsets into |
|
uint8 |
Concatenated UTF-8 label or prim-path bytes, without terminal NULs. |
For row i, decode label_data[label_offsets[i]:label_offsets[i+1]].
Do not use strlen: embedded NUL bytes are preserved. Keep repeated ID/path
pairs and sparse empty rows; multiple paths can share one ID, and semantic
class indices can address the original row positions. Empty outputs have
id.shape == (0, 4), label_offsets == [0] and no label bytes.
CPU and default mappings return CPU tensors. CUDA mappings return all three tensors on the GPU; CUDA-array mappings are unsupported. Follow the mapping’s CUDA synchronization contract before device access. Keep the mapping alive while using tensor views, or copy the values before unmapping.
The C example reuses sd_field() and sd_read() from
bounding-box decoding to read the named fields.
Accumulate delta pairs from the first delivered frame and merge subsequent additions
into that history. An empty delta after warmup is normal. Clear this history when
starting a different scene; deltas do not describe removals. Author these RenderVars
before scene start so initial IDs are allocated and delivered. SemanticIdMap
is a full class-label map; stable deltas associate geometry IDs with USD paths.
def decode_sd_identifiers(data: dict) -> list[tuple[tuple[int, ...], str]]:
"""Read UTF-8 spans without dropping repeated IDs or any of their four words."""
ids, offsets, storage = data["id"], data["label_offsets"], data["label_data"]
if (ids.dtype != np.uint32 or ids.shape != (len(ids), 4)
or offsets.dtype != np.uint64 or offsets.shape != (len(ids) + 1,)
or storage.dtype != np.uint8 or storage.ndim != 1 or offsets[0] != 0
or offsets[-1] != len(storage) or np.any(offsets[1:] < offsets[:-1])):
raise ValueError("Invalid identifier field shapes or string storage")
return [(tuple(int(word) for word in identifier), storage[int(start):int(end)].tobytes().decode("utf-8"))
for identifier, start, end in zip(ids, offsets[:-1], offsets[1:])]
// Decode one ID/path or ID/label pair. Keep repeated IDs as separate rows.
static bool sd_identifier(const ovrtx_render_var_output_t* output, size_t row, uint32_t id[4],
const char** label, size_t* length) {
const DLTensor* ids = sd_field(output, "id");
const DLTensor* offsets = sd_field(output, "label_offsets");
const DLTensor* data = sd_field(output, "label_data");
if (!ids || ids->ndim != 2 || !ids->shape || ids->shape[0] < 0 || ids->shape[1] != 4 ||
row >= size_t(ids->shape[0]) || !offsets || offsets->ndim != 1 || !offsets->shape ||
offsets->shape[0] != ids->shape[0] + 1 || !data || data->device.device_type != kDLCPU ||
data->ndim != 1 || !data->shape || data->shape[0] < 0 || data->dtype.code != kDLUInt ||
data->dtype.bits != 8 || data->dtype.lanes != 1 || (data->strides && data->strides[0] != 1) ||
!label || !length) return false;
uint64_t spans[2] = {};
if (!sd_read(ids, row * 4, 4, kDLUInt, 32, id) ||
!sd_read(offsets, row, 2, kDLUInt, 64, spans) || spans[0] > spans[1] ||
spans[1] > uint64_t(data->shape[0]) || (spans[1] && !data->data)) return false;
*length = size_t(spans[1] - spans[0]);
*label = data->data ? static_cast<const char*>(data->data) + data->byte_offset + spans[0] : "";
return true; // Read exactly length UTF-8 bytes; embedded NULs are permitted.
}
Stable-to-semantic pairs#
Applies to StableIdSemanticIdMap.
Schema StableIdSemanticIdMap version 0 exposes stable_id and semantic_id
as uint32 tensors of shape (N,4). Default/CPU mappings return CPU tensors;
explicit CUDA mappings return CUDA tensors. CUDA-array mapping is unsupported.
Respect DLPack strides and synchronize GPU access. Join all four words of each stable
ID to SemanticInstanceSegmentation and its semantic class ID
to SemanticIdMap. This maps labeled semantic instances rather than every raw
geometry ID in StableIdSegmentation. Preserve
one-to-many relationships rather than overwriting repeated keys. Zero IDs denote
background. SemanticInstanceSegmentation uses (0, 0, 0, 1) for unlabeled
geometry; SemanticSegmentation uses class index 1. Class indices occupy the
first word of the four-word class ID, so the corresponding class tuple is
(1, 0, 0, 0). Non-stable instance pixels use 0 for background and 1
for unlabeled geometry. For a labeled pixel, subtract two from its non-stable
instance value to index the pair-record array.
def decode_sd_id_pairs(data: dict) -> np.ndarray:
"""Join full-width IDs from named strided tensors, preserving repeated rows."""
stable, semantic = data["stable_id"], data["semantic_id"]
if stable.shape != (len(stable), 4) or semantic.shape != stable.shape:
raise ValueError("ID-pair fields must have matching (N,4) shapes")
result = np.empty(len(stable), dtype=[("stable_id", np.uint32, 4), ("semantic_id", np.uint32, 4)])
result["stable_id"], result["semantic_id"] = stable, semantic
return result
typedef struct SdIdPair {
uint32_t stable_id[4], semantic_id[4];
} SdIdPair;
static bool sd_id_pair(const ovrtx_render_var_output_t* output, size_t row, SdIdPair* value) {
const DLTensor* stable = sd_field(output, "stable_id");
const DLTensor* semantic = sd_field(output, "semantic_id");
return value && stable && semantic && stable->ndim == 2 && semantic->ndim == 2 &&
stable->shape && semantic->shape && stable->shape[1] == 4 && semantic->shape[1] == 4 &&
stable->shape[0] == semantic->shape[0] && stable->shape[0] >= 0 && row < size_t(stable->shape[0]) &&
sd_read(stable, row * 4, 4, kDLUInt, 32, value->stable_id) &&
sd_read(semantic, row * 4, 4, kDLUInt, 32, value->semantic_id);
}
// All four words belong to each ID, including its subset component.
Camera parameters#
Applies to CameraParams.
CameraParams has output-format version 0, no tensors, and 23 named CPU
parameters. These parameters remain on the CPU for every requested mapping
device. Read them from params and keep the mapping alive while using their
values. Each mapped result owns a snapshot of the camera that produced its frame.
view_matrix and projection_matrix are float32 arrays of shape (4,4).
The view matrix is world-to-camera; both matrices multiply row vectors.
Camera transforms and box transforms share scene units. The scalar
meters_per_scene_unit converts scene units to meters.
render_product_resolution is uint32 with shape (2,), and near_far
is float32 with shape (2,). camera_model is a uint8 array containing UTF-8
bytes without a terminator.
The remaining float32 parameters preserve the producer’s values and units:
focal_length, focal_distance, f_stop, fov_x_radians,
ftheta_width, ftheta_height, ftheta_cx, ftheta_cy,
ftheta_max_fov, ftheta_poly_a through ftheta_poly_f,
lens_p (2,) and lens_s (4,). The fields without a listed shape are
scalars with shape (1,).
The examples below read camera parameters and use them to project a box corner.
The C example reuses sd_field() and sd_read() from
bounding-box decoding to find parameters and read their
values with the declared DLPack strides.
def decode_sd_camera(data: dict) -> dict:
"""Use named CPU camera parameters; CameraParams has no output tensors."""
params = data["params"]
return {**params, "world_to_camera": params["view_matrix"], "projection": params["projection_matrix"],
"resolution": params["render_product_resolution"], "model": params["camera_model"].tobytes()}
// This is an application convenience type, not the producer's binary record layout.
typedef struct SdCamera {
float world_to_camera[4][4], projection[4][4];
uint32_t resolution[2];
char model[25];
} SdCamera;
static bool sd_camera(const ovrtx_render_var_output_t* output, SdCamera* camera) {
if (!output || output->num_tensors != 0 || !camera) return false;
const DLTensor* model = sd_field(output, "camera_model", true);
if (!model || model->ndim != 1 || !model->shape || model->shape[0] < 0 ||
size_t(model->shape[0]) >= sizeof(camera->model)) return false;
memset(camera, 0, sizeof(*camera));
return sd_read(sd_field(output, "view_matrix", true), 0, 16, kDLFloat, 32, camera->world_to_camera) &&
sd_read(sd_field(output, "projection_matrix", true), 0, 16, kDLFloat, 32, camera->projection) &&
sd_read(sd_field(output, "render_product_resolution", true), 0, 2, kDLUInt, 32, camera->resolution) &&
sd_read(model, 0, size_t(model->shape[0]), kDLUInt, 8, camera->model);
}
// All camera fields are CPU params, including focal_length, near_far and lens parameters.
// Matrices use row vectors. Box transforms and camera matrices share scene units.
Project a box corner into the image#
The decoded record is enough to draw a 3D box over the image. For a pinhole
camera, transform a local corner by the box’s local_to_world, then by
world_to_camera, then by projection, all as row vectors; divide by W;
and map the normalized device coordinates to pixels with Y inverted. A corner
behind the camera (W at or below zero) has no pixel: clip box edges against the
near plane before you draw them. Camera and box coordinates already share scene
units, so do not apply the meters-per-scene-unit scale to them.
The snippets below project one local point and return its pixel position, or nothing when the point lies behind the camera.
def project_sd_point(point, local_to_world, camera):
"""Project a local 3D point into pinhole pixels using row-vector matrices."""
if camera["model"].split(b"\0", 1)[0] != b"pinhole":
raise ValueError("This projection only handles pinhole cameras")
world = np.append(point, 1.0) @ local_to_world
clip = world @ camera["world_to_camera"] @ camera["projection"]
if clip[3] <= 0:
return None # Behind the camera; clip crossing box edges before drawing them.
ndc = clip[:2] / clip[3]
return (ndc * [0.5, -0.5] + 0.5) * camera["resolution"]
// Project one local point into pinhole pixels; behind-camera points need clipping.
static bool sd_project_point(const float point[3], const float local_to_world[4][4],
const SdCamera* camera, float pixel[2]) {
if (strncmp(camera->model, "pinhole", sizeof(camera->model)) != 0) return false;
float value[4] = {point[0], point[1], point[2], 1.0f};
const float (*matrices[3])[4] = {local_to_world, camera->world_to_camera, camera->projection};
for (size_t matrix = 0; matrix < 3; ++matrix) {
float next[4] = {};
for (size_t col = 0; col < 4; ++col)
for (size_t row = 0; row < 4; ++row)
next[col] += value[row] * matrices[matrix][row][col];
memcpy(value, next, sizeof(value));
}
if (value[3] <= 0) return false;
pixel[0] = (value[0] / value[3] * 0.5f + 0.5f) * camera->resolution[0];
pixel[1] = (-value[1] / value[3] * 0.5f + 0.5f) * camera->resolution[1];
return true;
}
Path Tracing pixels#
The Path Tracing table gives each tensor’s actual type and channel count.
Convert float16 to float32 before arithmetic. Divide uint8 color and roughness
by 255; float32 values keep their original units. The C reader accepts CPU,
contiguous mappings and accounts for byte_offset. Keep the mapping alive
while reading pixels.
Lighting outputs use RGB; normals and positions use XYZ. Motion uses XY pixel
displacement, with view-depth change in Z for four-channel screen-motion outputs.
MotionWorld uses scene units instead. Roughness uses the blue channel.
These values need the display
remapping described in the table; decoding alone does not tone-map them.
import numpy as np
def decode_pt_pixels(values: np.ndarray) -> np.ndarray:
"""Read typed PT pixels, converting half floats to float32 and UNORM bytes to [0,1]."""
if values.ndim != 3 or values.shape[-1] not in (1, 2, 4):
raise ValueError("Expected a channel-last PT image")
if values.dtype == np.uint8:
return values.astype(np.float32) / 255.0
if values.dtype not in (np.float16, np.float32):
raise ValueError("Expected uint8, float16, or float32 PT pixels")
return values.astype(np.float32)
// Decode IEEE binary16, including subnormal values and special exponents.
static float pt_half_to_float(uint16_t bits) {
const uint32_t exponent = (bits >> 10) & 31u;
const uint32_t fraction = bits & 1023u;
float value;
if (exponent == 31)
value = fraction ? std::numeric_limits<float>::quiet_NaN() : std::numeric_limits<float>::infinity();
else if (exponent == 0)
value = std::ldexp(static_cast<float>(fraction), -24);
else
value = std::ldexp(1.0f + static_cast<float>(fraction) / 1024.0f, static_cast<int>(exponent) - 15);
return bits & 0x8000u ? -value : value;
}
// Read one typed CPU pixel. Byte formats are UNORM; ID images need their uint32 decoder.
static bool pt_read_pixel(const DLTensor* tensor, int64_t x, int64_t y, float pixel[4]) {
const void* first = sd_image_data(tensor);
if (!first || !pixel || x < 0 || y < 0 || x >= tensor->shape[1] || y >= tensor->shape[0] ||
(tensor->shape[2] != 1 && tensor->shape[2] != 2 && tensor->shape[2] != 4)) return false;
const bool half = tensor->dtype.code == kDLFloat && tensor->dtype.bits == 16;
const bool single = tensor->dtype.code == kDLFloat && tensor->dtype.bits == 32;
const bool unorm = tensor->dtype.code == kDLUInt && tensor->dtype.bits == 8;
if (!half && !single && !unorm) return false;
const size_t channels = static_cast<size_t>(tensor->shape[2]);
const size_t offset = (static_cast<size_t>(y) * tensor->shape[1] + x) * channels;
const uint8_t* bytes = static_cast<const uint8_t*>(first);
for (size_t channel = 0; channel < 4; ++channel) {
pixel[channel] = 0;
if (channel >= channels) continue;
if (half) {
uint16_t bits = 0;
memcpy(&bits, bytes + (offset + channel) * 2, 2);
pixel[channel] = pt_half_to_float(bits);
} else if (single) {
memcpy(&pixel[channel], bytes + (offset + channel) * 4, 4);
} else pixel[channel] = bytes[offset + channel] / 255.0f;
}
return true;
}