Yes, that worked! Thanks. Now I encountered a couple of further problems.
First, it says that the function rep.orchestrator.step() does not exist. After looking at the docs (link), I changed that line to rep.orchestrator.preview() and it seems to work, let me know if this is correct.
Second, when I do rgb_data = rgb.get_data(device="gpu"), I get an error, saying again that
'Annotator' object has no attribute 'get_data'
I checked the source code in the isaac-sim folder, and in fact that function is commented out. I see there is a get_node() function, could we maybe use that instead?
As a reference, I paste here the Annotator class from the Isaac 2022.1.0 release, where the function get_data() is in fact commented out:
class Annotator:
""" Annotator class
Annotator instances identify the annotator name, it's initialization parameters, the render products it is tied to,
as well as the name of the OmniGraph template.
Initialization parameters can be overridden with `initialize()`, and render products can be set with `attach()`.
"""
def __init__(
self,
name: str,
init_params: dict = None,
render_product_idxs: List[int] = None,
device: str = None,
render_products: list = None,
template_name: str = None,
) -> None:
self._name = name
self._template_name = name if template_name is None else template_name
self._node_path = None
self._semantic_types = None
if (
self._template_name not in SyntheticData._ogn_templates_registry
and self._template_name not in SyntheticData._ogn_rendervars
):
raise AnnotatorRegistryError(f"The annotator `{name}` is missing from the annotator registry")
if init_params is None:
init_params = {}
if isinstance(init_params, dict):
self.initialize(**init_params)
else:
raise AnnotatorRegistryError(f"The value for `init_params` must be a a dict, got {type(init_params)}")
if render_product_idxs is None:
render_product_idxs = [0]
elif isinstance(render_product_idxs, int):
render_product_idxs = [render_product_idxs]
elif (not isinstance(render_product_idxs, list) and not isinstance(render_product_idxs, tuple)) or not all(
[isinstance(rpi, int) for rpi in render_product_idxs]
):
raise AnnotatorRegistryError(
f"The value for `render_product_idxs` must be a list of ints, got {render_product_idxs}"
)
self._render_product_idxs = render_product_idxs
if device is None:
self._device = "cpu"
elif isinstance(device, str) and device.lower() in ["cpu", "gpu"]:
self._device = device.lower()
else:
raise AnnotatorRegistryError(f"Invalid device `{device}` specified. Device must be one of ['cpu', 'gpu']")
# TODO need to be able to output arrays from annotator, not just from render vars
# if self._device == "cpu":
# SyntheticData.register_node_template(
# "omni.syntheticdata.SdRenderVarToRawArray",
# [[self._name, 0, {}], ["PostProcessDispatch", 0, {}]],
# f"{self._name}ExportRawArray",
# SyntheticData.StagePostProcess,
# )
if render_products is not None:
self.attach(render_products)
def initialize(self, **kwargs):
""" Initialize annotator parameters
The initialization parameters of the annotator. Initialize can only be called before the annotator has been
attached.
"""
if self._node_path is not None:
raise AnnotatorRegistryError("Annotator parameters cannot be initialized, it is already attached.")
if "semanticTypes" in kwargs:
self._semantic_types = kwargs["semanticTypes"]
self._init_params = {f"inputs:{k}": v for k, v in kwargs.items()}
def attach(self, render_products: List = None):
""" Attach annotator to specified render products.
Creates the OmniGraph nodes and connections.
Args:
render_products: List of render products to attach the annotator to.
"""
if self._node_path is not None:
raise AnnotatorRegistryError("Annotator is already attached.")
if render_products is not None:
if isinstance(render_products, str):
render_products = [render_products]
if len(render_products) < max(self._render_product_idxs):
raise AnnotatorRegistryError(
f"Expected at least {max(self._render_product_idxs)} render products, received only {len(render_products)}"
)
render_products = [render_products[rpi] for rpi in self._render_product_idxs]
sdg_iface = SyntheticData.Get()
# HACK to work around SyntheticData setup
template_name = self._template_name
if template_name in SyntheticData._ogn_rendervars:
template_name = template_name + "ExportRawArray"
sdg_iface.activate_node_template(
template_name,
0,
render_products,
attributes=self._init_params
# template_name, self._render_product_idxs, render_products, attributes=self._init_params
) # TODO `0` idx needs to be more flexible
# If output to CPU, add node to expose GPU array to CPU
# TODO
# if self._device == "cpu":
# sdg_iface.activate_node_template(self._name + "ExportRawArray", 0, render_products) # TODO `0` idx needs to be more flexible
render_product = render_products[0] if render_products else None # TODO support more than one render products
self._node_path = sdg_iface._get_node_path(template_name, render_product)
if self._semantic_types:
stage = omni.usd.get_context().get_stage()
nodePath = SyntheticData._get_node_path(
"InstanceMappingPre", None
) # TODO support more than one render products)
if stage.GetPrimAtPath(nodePath):
im_pre_node = og.Controller().node(nodePath)
# Accumulate allowed semantic types
current_types = set(
og.ContextHelper().get_attr_value(im_pre_node.get_attribute("inputs:types"))
) # Need to use context helper to access token array
new_types = list(current_types.union(self._semantic_types))
SyntheticData.Get().set_node_attributes("InstanceMappingPre", {"inputs:types": new_types}, None)
# node = self.get_node()
# node.create_attribute("outputs:annotator_name", og.Type(og.BaseDataType.TOKEN))
# node.get_attribute("outputs:annotator_name").set(self._name)
def is_attached(self):
return self._node_path is not None
def get_name(self):
return self._name
# async def get_data(self) -> np.ndarray:
# """ Get annotator data
# """
# # TODO(jiehan) fix get_data
# if not self._node_path:
# raise AnnotatorRegistryError(f"Unable to get data, annotator is not yet attached to render product(s)")
# # TODO await for sensor data to be available
# attributes_data = {"outputs:data": None}
# if is2DArray:
# attributes_data["outputs:width"] = None
# attributes_data["outputs:height"] = None
# else:
# attributes_data["outputs:bufferSize"] = None
# rendervar_name = SyntheticData.convert_sensor_type_to_rendervar(sensor_type.name)
# get_synthetic_data().get_node_attributes(
# rendervar_name + "ExportRawArray", attributes_data, viewport.get_render_product_path()
# )
# data = attributes_data["outputs:data"]
# return data
# TODO array type needs to be stored in node
# if height and width:
# height = attributes_data["outputs:height"]
# width = attributes_data["outputs:width"]
# bufferSize = height*width*elemCount*np.dtype(elemType).itemsize
# else:
# bufferSize = attributes_data["outputs:bufferSize"]
# if (data is None) or (len(data) < np.dtype(elemType).itemsize):
# if is2DArray:
# shape = (0, 0, elemCount) if elemCount > 1 else (0, 0)
# else:
# shape = (0, elemCount) if elemCount > 1 else (0)
# return np.empty(shape, elemType)
# assert bufferSize == len(data)
# data = data.view(elemType)
# assert len(data) > 0
# if not is2DArray:
# return data.reshape(data.shape[0] // elemCount, elemCount) if elemCount > 1 else data
# return data.reshape(height, width, elemCount) if elemCount > 1 else data.reshape(height, width)
def get_node(self):
return SyntheticData.Get()._graphNodes[self._node_path]