Decoding camera outputs#
Image tensors#
Applies to NormalSD, DepthSD, Depth, XRDepth, DistanceToCameraSD, DistanceToImagePlaneSD, DiffuseAlbedoSD, Camera3dPositionSD, TargetMotionSD, SemanticSegmentation, SemanticInstanceSegmentation, StableIdSegmentation, and NonStableInstanceSegmentation.
Image tensors already carry their scalar type and shape in DLPack. Python preserves
that type; C callers inspect dtype.code, dtype.bits, and shape before
reading pixels. The C helper checks CPU placement and contiguous layout and adds
byte_offset. Keep mappings alive while using their pointers.
Float images use float32; albedo uses uint8. Segmentation images use uint32: keep all four words for stable and semantic IDs. Color IDs only for display; do not normalize them before joining records. Non-stable IDs are runtime indices. Map normals from [-1, 1] to display colors, normalize finite distances for display, and encode motion direction as hue and displacement magnitude as brightness. The table gives each output’s units and channels.
def map_sd_image(frame, render_var_path: str) -> np.ndarray:
"""Copy a typed image to CPU memory before its mapping is released."""
mapped = frame.render_vars[render_var_path].map(device=ovrtx.Device.CPU)
return np.from_dlpack(mapped).copy()
// The output must already be mapped with OVRTX_MAP_DEVICE_TYPE_CPU.
// Return the first pixel only for contiguous, channel-last image tensors.
static const void* sd_image_data(const DLTensor* tensor) {
if (!tensor || tensor->device.device_type != kDLCPU || tensor->ndim != 3 ||
tensor->dtype.lanes != 1 || !tensor->data || !tensor->shape) return nullptr;
if (tensor->strides && (tensor->strides[2] != 1 ||
tensor->strides[1] != tensor->shape[2] ||
tensor->strides[0] != tensor->shape[1] * tensor->shape[2])) return nullptr;
return static_cast<const uint8_t*>(tensor->data) + tensor->byte_offset;
}
Packed record buffers#
Buffer outputs map to one-dimensional uint8 tensors. In shape (B,),
B is the byte length, not the record count. Decode the little-endian bytes
using the field types shown below; do not normalize them as image pixels.
The helpers validate buffer lengths before reading records. In C, copy records
with memcpy() to avoid alignment assumptions about the mapped pointer.
counted=true reads the final uint32 count; uncounted arrays use the byte length.
Semantic boxes reserve unused capacity: read only the populated records.
def decode_sd_records(data: np.ndarray, dtype: np.dtype, counted: bool = False) -> np.ndarray:
"""Decode little-endian records, checking lengths before reading reserved buffers."""
buffer = np.ascontiguousarray(data).view(np.uint8).reshape(-1)
if counted:
# Semantic boxes reserve unused capacity; their final uint32 is the actual count.
if buffer.size < 4:
raise ValueError("Missing record count")
count = int.from_bytes(buffer[-4:].tobytes(), "little")
if count > (buffer.size - 4) // dtype.itemsize:
raise ValueError("Record count exceeds buffer capacity")
else:
if buffer.size % dtype.itemsize:
raise ValueError("Incomplete record")
count = buffer.size // dtype.itemsize
return buffer[:count * dtype.itemsize].view(dtype)
typedef struct SdRecordView {
const uint8_t* data;
size_t count;
size_t stride;
} SdRecordView;
// Read little-endian buffer lengths before copying records with memcpy().
// counted is true for semantic boxes and identifier maps, whose final word is a count.
static bool sd_records(const uint8_t* data, size_t bytes, size_t stride,
bool counted, SdRecordView* output) {
if (!data || !stride || !output) return false;
size_t count = 0;
if (counted) {
if (bytes < 4) return false;
uint32_t stored_count = 0;
memcpy(&stored_count, data + bytes - 4, 4);
count = stored_count;
if (count > (bytes - 4) / stride) return false;
} else {
if (bytes % stride) return false;
count = bytes / stride;
}
*output = {data, count, stride};
return true;
}
Bounding boxes#
Applies to BoundingBox2DTightSD, BoundingBox2DLooseSD, BoundingBox3DSD, SemanticBoundingBox2DTight, SemanticBoundingBox2DLoose, and SemanticBoundingBox3D.
Use semantic=True for semantic boxes and three_d=True for 3D boxes in Python.
In C, pass the corresponding structure size to sd_records() and set counted
only for semantic boxes. Skip records whose maximum is below their minimum.
2D coordinates are pixel bounds; draw rectangles over the color image. Raw boxes
join visibility and token records by instance/tile ID. Semantic boxes carry full
128-bit instance and class IDs.
Each 3D bounding-box record contains local bounds and a 4×4 transform
matrix from local space to world space. The decoders below name this field
local_to_world. Use the matrix from the same record to transform the eight
box corners before projecting into the camera.
def decode_sd_boxes(data: np.ndarray, semantic: bool = False, three_d: bool = False) -> np.ndarray:
"""Decode raw or semantic boxes; 3D bounds are local and include a world transform."""
fields = [("stable_id", "<u4", 4), ("semantic_id", "<u4", 4)] if semantic else [
("instance_id", "<u4"), ("tile_id", "<u4")]
fields += [("min", "<f4" if three_d else "<i4", 3 if three_d else 2),
("max", "<f4" if three_d else "<i4", 3 if three_d else 2)]
if three_d:
fields += [("local_to_world", "<f4", (4, 4))]
return decode_sd_records(data, np.dtype(fields), counted=semantic)
typedef struct SdBox2D {
uint32_t instance_id, tile_id;
int32_t min[2], max[2];
} SdBox2D;
typedef struct SdBox3D {
uint32_t instance_id, tile_id;
float min[3], max[3], local_to_world[4][4];
} SdBox3D;
typedef struct SdSemanticBox2D {
uint32_t stable_id[4], semantic_id[4];
int32_t min[2], max[2];
} SdSemanticBox2D;
typedef struct SdSemanticBox3D {
uint32_t stable_id[4], semantic_id[4];
float min[3], max[3], local_to_world[4][4];
} SdSemanticBox3D;
// Call sd_records(data, bytes, sizeof(record_type), semantic, &view).
// Copy record i with memcpy(&box, view.data + i * view.stride, sizeof(box)).
// Bounds with max < min are invalid; clip 3D box edges before projecting them.
Occlusion#
Applies to OcclusionSD.
Read 12-byte records without a trailing count. Join each instance/tile key to its raw bounding box, then annotate that box with the ratio. Skip negative ratios; valid ratios range from zero (unoccluded) to one (fully occluded).
def decode_sd_occlusion(data: np.ndarray) -> np.ndarray:
"""Read occlusion ratios with their instance/tile keys; negative ratios are invalid."""
dtype = np.dtype([("instance_id", "<u4"), ("tile_id", "<u4"), ("ratio", "<f4")])
return decode_sd_records(data, dtype)
typedef struct SdOcclusion {
uint32_t instance_id, tile_id;
float ratio;
} SdOcclusion;
// Use sd_records(..., sizeof(SdOcclusion), false, &view).
// Join to raw boxes by instance_id AND tile_id; negative ratios are invalid.
Instance token map#
Applies to InstanceIdTokenMapSD.
Read 24-byte records without a trailing count. The first record is a header:
instance_id is the number of following records; the low/high uint32 words of
prim_token give the minimum/maximum instance ID; prototype gives the tile
count. Subsequent records contain an opaque runtime token, prototype index, and
instance index. The token is not a string address or a persistent network ID.
def decode_sd_tokens(data: np.ndarray) -> tuple[np.void, np.ndarray]:
"""Separate the token-map header from aligned instance records."""
dtype = np.dtype({"names": ["instance_id", "prim_token", "prototype", "instance_index"],
"formats": ["<u4", "<u8", "<u4", "<u4"],
"offsets": [0, 8, 16, 20], "itemsize": 24})
records = decode_sd_records(data, dtype)
if not len(records) or int(records[0]["instance_id"]) != len(records) - 1:
raise ValueError("Token-map header count does not match records")
return records[0], records[1:]
typedef struct SdInstanceToken {
uint32_t instance_id, padding;
uint64_t prim_token;
uint32_t prototype, instance_index;
} SdInstanceToken;
// Use sd_records(..., sizeof(SdInstanceToken), false, &view).
// Record 0 is a header: instance_id is the number of following records,
// prim_token packs minInstanceId (low 32 bits) and maxInstanceId (high 32 bits),
// and prototype is the tile count. Copy with memcpy(), not an unaligned cast.
// prim_token is an opaque runtime handle, not an address to a string.
Instance map#
Applies to InstanceMap.
Read uint32 semantic indices without a trailing count. Element i corresponds
to minInstanceId + i from the token-map header. Use these indices with the
non-stable instance/semantic outputs; they are not the four-word stable IDs.
def decode_sd_instance_map(data: np.ndarray) -> np.ndarray:
"""Read semantic indices; element i corresponds to minInstanceId + i from the token header."""
return decode_sd_records(data, np.dtype("<u4"))
// InstanceMap is an array of uint32 semantic indices without a trailing count.
static bool sd_instance_map(const uint8_t* data, size_t bytes, SdRecordView* view) {
return sd_records(data, bytes, sizeof(uint32_t), false, view);
}
// Element i belongs to minInstanceId + i from the InstanceIdTokenMapSD header.
Identifier maps and deltas#
Applies to SemanticIdMap, SemanticIdMapDeltas, StableIdMap, and StableIdMapDeltas.
Each record contains four uint32 ID words, a UTF-8 byte length, and a byte offset
from the start of the buffer. Strings follow the records; the final uint32 gives
the record count. Read exactly the declared byte length: do not use strlen
or require a NUL terminator. Keep repeated ID/path pairs; multiple paths can share one ID.
Accumulate delta pairs from the first delivered frame and merge subsequent additions
into that history. An empty delta after warmup is normal. Clear this history when
starting a different scene; deltas do not describe removals. Author these RenderVars
before scene start so initial IDs are allocated and delivered. SemanticIdMap
is a full class-label map; stable deltas associate geometry IDs with USD paths.
def decode_sd_identifiers(data: np.ndarray) -> list[tuple[tuple[int, ...], str]]:
"""Decode labels/paths without dropping repeated IDs or any of their four words."""
buffer = np.ascontiguousarray(data).view(np.uint8).reshape(-1)
dtype = np.dtype([("id", "<u4", 4), ("length", "<u4"), ("offset", "<u4")])
records = decode_sd_records(buffer, dtype, counted=True)
labels = []
for record in records:
offset, length = int(record["offset"]), int(record["length"])
if offset < records.nbytes or offset + length > buffer.size - 4:
raise ValueError("Identifier string extends outside string storage")
labels.append((tuple(int(word) for word in record["id"]),
buffer[offset:offset + length].tobytes().decode("utf-8")))
return labels
typedef struct SdIdentifier {
uint32_t id[4], length, offset;
} SdIdentifier;
// Decode one ID/path or ID/label pair. Keep repeated IDs as separate records.
static bool sd_identifier(const uint8_t* data, size_t bytes, size_t index,
SdIdentifier* record, const char** label) {
SdRecordView view{};
if (!record || !label || !sd_records(data, bytes, sizeof(SdIdentifier), true, &view) ||
index >= view.count) return false;
memcpy(record, view.data + index * view.stride, sizeof(*record));
if (record->offset < view.count * view.stride || record->offset >= bytes - 4 ||
record->length > bytes - 4 - record->offset) return false;
*label = reinterpret_cast<const char*>(data + record->offset);
return true; // Read record->length UTF-8 bytes; do not assume a NUL terminator.
}
Stable-to-semantic pairs#
Applies to StableIdSemanticIdMap.
Read 32-byte records without a trailing count. Join all four words of each stable
ID to SemanticInstanceSegmentation and its semantic class ID
to SemanticIdMap. This maps labeled semantic instances rather than every raw
geometry ID in StableIdSegmentation. Preserve
one-to-many relationships rather than overwriting repeated keys. Zero IDs denote
background. SemanticInstanceSegmentation uses (0, 0, 0, 1) for unlabeled
geometry; SemanticSegmentation uses class index 1. Class indices occupy the
first word of the four-word class ID, so the corresponding class tuple is
(1, 0, 0, 0). Non-stable instance pixels use 0 for background and 1
for unlabeled geometry. For a labeled pixel, subtract two from its non-stable
instance value to index the pair-record array.
def decode_sd_id_pairs(data: np.ndarray) -> np.ndarray:
"""Read stable-to-semantic ID pairs; this buffer has no trailing count."""
dtype = np.dtype([("stable_id", "<u4", 4), ("semantic_id", "<u4", 4)])
return decode_sd_records(data, dtype)
typedef struct SdIdPair {
uint32_t stable_id[4], semantic_id[4];
} SdIdPair;
// Use sd_records(..., sizeof(SdIdPair), false, &view); there is no trailing count.
// All four words belong to each ID, including its subset component.
Camera parameters#
Applies to CameraParams.
Read exactly one 260-byte record, including the three padding bytes after the 25-byte model name. The first matrix is world-to-camera; both matrices multiply row vectors. Camera matrices and box transforms share scene units. The scale field converts scene units to meters. Projection model parameters follow the matrices and image resolution.
def decode_sd_camera(data: np.ndarray) -> np.void:
"""Decode the 260-byte camera record, including the projection-model padding."""
dtype = np.dtype([
("world_to_camera", "<f4", (4, 4)), ("projection", "<f4", (4, 4)),
("meters_per_scene_unit", "<f4"), ("resolution", "<u4", 2), ("near_far", "<f4", 2),
("model", "S25"), ("padding", "u1", 3),
("focal_length", "<f4"), ("focal_distance", "<f4"), ("f_stop", "<f4"), ("fov_x", "<f4"),
("fisheye_width", "<f4"), ("fisheye_height", "<f4"), ("fisheye_center", "<f4", 2),
("fisheye_max_fov", "<f4"), ("fisheye_polynomial", "<f4", 6),
("lens_p", "<f4", 2), ("lens_s", "<f4", 4)])
records = decode_sd_records(data, dtype)
if len(records) != 1:
raise ValueError("Expected one camera record")
return records[0]
typedef struct SdCamera {
float world_to_camera[4][4], projection[4][4], meters_per_scene_unit;
uint32_t resolution[2];
float near_far[2];
char model[25];
uint8_t padding[3];
float focal_length, focal_distance, f_stop, fov_x;
float fisheye_width, fisheye_height, fisheye_center[2], fisheye_max_fov;
float fisheye_polynomial[6], lens_p[2], lens_s[4];
} SdCamera;
// Expect exactly sizeof(SdCamera) bytes; copy with memcpy(&camera, data, sizeof(camera)).
// Matrices use row vectors. Box transforms and camera matrices share scene units.
Path Tracing pixels#
The Path Tracing table gives each tensor’s actual type and channel count.
Convert float16 to float32 before arithmetic. Divide uint8 color and roughness
by 255; float32 values keep their original units. The C reader accepts CPU,
contiguous mappings and accounts for byte_offset. Keep the mapping alive
while reading pixels.
Lighting outputs use RGB; normals and positions use XYZ. Motion uses XY pixel
displacement, with view-depth change in Z for four-channel screen-motion outputs.
MotionWorld uses scene units instead. Roughness uses the blue channel.
These values need the display
remapping described in the table; decoding alone does not tone-map them.
def decode_pt_pixels(values: np.ndarray) -> np.ndarray:
"""Read typed PT pixels, converting half floats to float32 and UNORM bytes to [0,1]."""
if values.ndim != 3 or values.shape[-1] not in (1, 2, 4):
raise ValueError("Expected a channel-last PT image")
if values.dtype == np.uint8:
return values.astype(np.float32) / 255.0
if values.dtype not in (np.float16, np.float32):
raise ValueError("Expected uint8, float16, or float32 PT pixels")
return values.astype(np.float32)
// Decode IEEE binary16, including subnormal values and special exponents.
static float pt_half_to_float(uint16_t bits) {
const uint32_t exponent = (bits >> 10) & 31u;
const uint32_t fraction = bits & 1023u;
float value;
if (exponent == 31)
value = fraction ? std::numeric_limits<float>::quiet_NaN() : std::numeric_limits<float>::infinity();
else if (exponent == 0)
value = std::ldexp(static_cast<float>(fraction), -24);
else
value = std::ldexp(1.0f + static_cast<float>(fraction) / 1024.0f, static_cast<int>(exponent) - 15);
return bits & 0x8000u ? -value : value;
}
// Read one typed CPU pixel. Byte formats are UNORM; ID images need their uint32 decoder.
static bool pt_read_pixel(const DLTensor* tensor, int64_t x, int64_t y, float pixel[4]) {
const void* first = sd_image_data(tensor);
if (!first || !pixel || x < 0 || y < 0 || x >= tensor->shape[1] || y >= tensor->shape[0] ||
(tensor->shape[2] != 1 && tensor->shape[2] != 2 && tensor->shape[2] != 4)) return false;
const bool half = tensor->dtype.code == kDLFloat && tensor->dtype.bits == 16;
const bool single = tensor->dtype.code == kDLFloat && tensor->dtype.bits == 32;
const bool unorm = tensor->dtype.code == kDLUInt && tensor->dtype.bits == 8;
if (!half && !single && !unorm) return false;
const size_t channels = static_cast<size_t>(tensor->shape[2]);
const size_t offset = (static_cast<size_t>(y) * tensor->shape[1] + x) * channels;
const uint8_t* bytes = static_cast<const uint8_t*>(first);
for (size_t channel = 0; channel < 4; ++channel) {
pixel[channel] = 0;
if (channel >= channels) continue;
if (half) {
uint16_t bits = 0;
memcpy(&bits, bytes + (offset + channel) * 2, 2);
pixel[channel] = pt_half_to_float(bits);
} else if (single) {
memcpy(&pixel[channel], bytes + (offset + channel) * 4, 4);
} else pixel[channel] = bytes[offset + channel] / 255.0f;
}
return true;
}