Semantic Labels#
Semantic labels are USD SemanticsAPI metadata used by annotation outputs
such as semantic segmentation. In ovrtx, labels can be authored directly in USD
or layered over an existing scene with inline USDA.
Runtime Overrides#
Use ordinary attribute writes to update semantic class and label metadata after the stage is loaded.
SEMANTIC_CLASS_USDA = f"""#usda 1.0
(
subLayers = [
@{SEMANTIC_LABELS_PATH}@,
@{TEST_BASE_PATH}@
]
)
def "Render"
{{
def RenderProduct "SemanticCamera"
{{
int2 resolution = {RESOLUTION}
rel camera = </World/Camera>
rel orderedVars = [<SemanticSegmentation>, <SemanticIdMap>]
def RenderVar "SemanticSegmentation"
{{
string sourceName = "SemanticSegmentation"
}}
def RenderVar "SemanticIdMap"
{{
string sourceName = "SemanticIdMap"
}}
}}
}}
"""
std::string usda = "#usda 1.0\n"
"(\n"
" subLayers = [\n"
" @" + semantic_labels_path + "@,\n"
" @" + scene_path + "@\n"
" ]\n"
")\n\n"
"def \"Render\"\n"
"{\n"
" def RenderProduct \"SemanticCamera\"\n"
" {\n"
" int2 resolution = (1280, 720)\n"
" rel camera = </World/Camera>\n"
" rel orderedVars = [<SemanticSegmentation>, <SemanticIdMap>]\n"
"\n"
" def RenderVar \"SemanticSegmentation\"\n"
" {\n"
" string sourceName = \"SemanticSegmentation\"\n"
" }\n"
"\n"
" def RenderVar \"SemanticIdMap\"\n"
" {\n"
" string sourceName = \"SemanticIdMap\"\n"
" }\n"
" }\n"
"}\n";
ovstage_population_enqueue_result_t pr = ovstage_population_open_usd_from_string(
stage_,
{usda.c_str(), usda.size()},
/*ordinal=*/1,
/*time=*/NAN,
OVSTAGE_POPULATION_DOMAIN_RENDERING);
ASSERT_EQ(pr.status, OVSTAGE_OK) << format_ovstage_population_last_error();
docs_wait_ovstage_population_no_errors(stage_, pr.op_index);
docs_ovstage_advance_write_floor(stage_, 1);
Interpreting Segmentation Output#
Semantic segmentation output maps pixels to numeric semantic identifiers. Use the identifier information in the render output to map those ids back to labels.
def _map_render_var(frame, name: str) -> np.ndarray | dict[str, np.ndarray]:
with frame.render_vars[f"/Render/SemanticCamera/{name}"].map(device=ovrtx.Device.CPU) as mapped:
if name != "SemanticIdMap":
return np.from_dlpack(mapped).copy()
return {key: np.from_dlpack(mapped[key]).copy() for key in mapped}
def _decode_semantic_id_map(fields: dict[str, np.ndarray]) -> dict[tuple[int, ...], str]:
ids, offsets, data = fields["id"], fields["label_offsets"], fields["label_data"]
if (ids.dtype != np.uint32 or ids.shape != (len(ids), 4)
or offsets.dtype != np.uint64 or offsets.shape != (len(ids) + 1,)
or data.dtype != np.uint8 or data.ndim != 1 or offsets[0] != 0
or offsets[-1] != len(data) or np.any(offsets[1:] < offsets[:-1])):
raise ValueError("Invalid semantic identifier fields or label spans")
return {tuple(int(word) for word in identifier): data[int(start):int(end)].tobytes().decode("utf-8").rstrip()
for identifier, start, end in zip(ids, offsets[:-1], offsets[1:])}
def test_semantic_class_labels_are_rendered(renderer, stage):
"""Set semantic class labels and verify their rendered semantic IDs."""
ordinal = 1
ovstage.population.open_usd_from_string(stage, SEMANTIC_CLASS_USDA, ordinal=ordinal)
stage.advance_write_floor(ordinal, ovstage.Scope.ALL).wait()
for _ in range(5):
renderer.step(render_products={RENDER_PRODUCT_PATH}, delta_time=1.0 / 60.0, ordinal=ordinal)
products = renderer.step(render_products={RENDER_PRODUCT_PATH}, delta_time=1.0 / 60.0, ordinal=ordinal)
frame = products[RENDER_PRODUCT_PATH].frames[0]
semantic_id_map = _decode_semantic_id_map(_map_render_var(frame, "SemanticIdMap"))
ids_by_label = {label: semantic_id for semantic_id, label in semantic_id_map.items()}
logo_id = ids_by_label["class: logo;"]
ground_id = ids_by_label["class: ground;"]
semantic_segmentation = np.squeeze(_map_render_var(frame, "SemanticSegmentation"))
# Scalar class pixels occupy the first word; preserve all four words when joining the map.
semantic_ids_in_image = {(int(value), 0, 0, 0) for value in np.unique(semantic_segmentation)}
assert logo_id in semantic_ids_in_image
assert ground_id in semantic_ids_in_image
assert np.count_nonzero(semantic_segmentation == logo_id[0]) > 0
assert np.count_nonzero(semantic_segmentation == ground_id[0]) > 0
static std::map<std::array<uint32_t, 4>, std::string> decode_semantic_id_map(ovrtx_render_var_output_t const& output) {
auto field = [&](char const* name, uint8_t bits, int ndim) -> DLTensor const& {
for (size_t i = 0; i < output.num_tensors; ++i) {
auto const& entry = output.tensors[i];
if (!entry.name || !entry.name->ptr || entry.name->length != strlen(name) ||
memcmp(entry.name->ptr, name, entry.name->length)) continue;
if (!entry.dl || entry.dl->device.device_type != kDLCPU || entry.dl->dtype.code != kDLUInt ||
entry.dl->dtype.bits != bits || entry.dl->dtype.lanes != 1 || entry.dl->ndim != ndim ||
!entry.dl->shape) throw std::invalid_argument("Invalid semantic identifier field");
for (int dim = 0; dim < ndim; ++dim)
if (entry.dl->shape[dim] < 0 || (entry.dl->strides && entry.dl->strides[dim] < 0))
throw std::invalid_argument("Invalid semantic identifier shape or stride");
return *entry.dl;
}
throw std::invalid_argument("Missing named semantic identifier field");
};
DLTensor const& ids = field("id", 32, 2);
DLTensor const& offsets = field("label_offsets", 64, 1);
DLTensor const& data = field("label_data", 8, 1);
if (ids.shape[1] != 4 || offsets.shape[0] < 1 || offsets.shape[0] - 1 != ids.shape[0] ||
(ids.shape[0] && !ids.data) || !offsets.data || (data.shape[0] && !data.data) ||
(data.strides && data.strides[0] != 1))
throw std::invalid_argument("Invalid semantic identifier field shapes");
auto offset = [&](int64_t index) {
uint64_t value;
size_t const stride = offsets.strides ? size_t(offsets.strides[0]) : 1;
memcpy(&value, static_cast<uint8_t const*>(offsets.data) + offsets.byte_offset +
size_t(index) * stride * sizeof(value), sizeof(value));
return value;
};
if (offset(0) != 0 || offset(ids.shape[0]) != uint64_t(data.shape[0]))
throw std::invalid_argument("Invalid semantic label spans");
std::map<std::array<uint32_t, 4>, std::string> labels_by_id;
for (int64_t i = 0; i < ids.shape[0]; ++i) {
uint64_t const start = offset(i), end = offset(i + 1);
if (start > end || end > uint64_t(data.shape[0]))
throw std::invalid_argument("Invalid semantic label spans");
std::array<uint32_t, 4> id{};
for (size_t word = 0; word < 4; ++word) {
size_t const index = size_t(i) * (ids.strides ? size_t(ids.strides[0]) : 4) +
word * (ids.strides ? size_t(ids.strides[1]) : 1);
memcpy(&id[word], static_cast<uint8_t const*>(ids.data) + ids.byte_offset + index * sizeof(uint32_t), 4);
}
char const* text = data.data ? static_cast<char const*>(data.data) + data.byte_offset + start : "";
std::string label(text, size_t(end - start));
trim_semantic_label(label);
labels_by_id[id] = label;
}
return labels_by_id;
}
static std::set<std::array<uint32_t, 4>> collect_semantic_segmentation_ids(DLTensor const& tensor) {
EXPECT_EQ(tensor.dtype.bits, 32);
EXPECT_EQ(tensor.dtype.lanes, 1);
EXPECT_TRUE(tensor.ndim == 2 || tensor.ndim == 3);
if (tensor.ndim == 3) {
EXPECT_EQ(tensor.shape[2], 1);
}
int64_t const height = tensor.shape[0];
int64_t const width = tensor.shape[1];
int64_t const stride_y = tensor.strides
? tensor.strides[0]
: width * (tensor.ndim == 3 ? tensor.shape[2] : 1);
int64_t const stride_x = tensor.strides
? tensor.strides[1]
: (tensor.ndim == 3 ? tensor.shape[2] : 1);
uint8_t const* base = static_cast<uint8_t const*>(tensor.data) + tensor.byte_offset;
// Scalar class pixels occupy the first word of the full semantic map ID.
std::set<std::array<uint32_t, 4>> ids;
if (tensor.dtype.code == kDLUInt) {
uint32_t const* values = reinterpret_cast<uint32_t const*>(base);
for (int64_t y = 0; y < height; ++y) {
for (int64_t x = 0; x < width; ++x) {
ids.insert({values[y * stride_y + x * stride_x], 0, 0, 0});
}
}
} else if (tensor.dtype.code == kDLInt) {
int32_t const* values = reinterpret_cast<int32_t const*>(base);
for (int64_t y = 0; y < height; ++y) {
for (int64_t x = 0; x < width; ++x) {
ids.insert({static_cast<uint32_t>(values[y * stride_y + x * stride_x]), 0, 0, 0});
}
}
} else {
ADD_FAILURE() << "SemanticSegmentation must use 32-bit integer pixels";
}
return ids;
}
Notes#
Put labels on the prims whose pixels or hits should carry that semantic meaning.
Inline override layers are useful when a source asset cannot be edited.
For sensor material behavior, use Non-Visual Sensor Materials instead. Semantic labels and non-visual material labels serve different systems.