From ef009cb7c4d33b3423067f3f1cb4f61006936b13 Mon Sep 17 00:00:00 2001 From: James Tigue Date: Mon, 14 Apr 2025 13:30:16 -0400 Subject: [PATCH 01/12] add rtx lidar --- apps/isaaclab.python.headless.rendering.kit | 6 + apps/isaaclab.python.rendering.kit | 6 + .../isaaclab/sensors/rtx_lidar/__init__.py | 8 + .../isaaclab/sensors/rtx_lidar/rtx_lidar.py | 342 ++++++++++++++++++ .../sensors/rtx_lidar/rtx_lidar_cfg.py | 56 +++ .../sensors/rtx_lidar/rtx_lidar_data.py | 64 ++++ .../isaaclab/sim/spawners/sensors/__init__.py | 4 +- .../isaaclab/sim/spawners/sensors/sensors.py | 78 +++- .../sim/spawners/sensors/sensors_cfg.py | 82 ++++- .../isaaclab/test/sensors/test_rtx_lidar.py | 260 +++++++++++++ 10 files changed, 902 insertions(+), 4 deletions(-) create mode 100644 source/isaaclab/isaaclab/sensors/rtx_lidar/__init__.py create mode 100644 source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py create mode 100644 source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py create mode 100644 source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_data.py create mode 100644 source/isaaclab/test/sensors/test_rtx_lidar.py diff --git a/apps/isaaclab.python.headless.rendering.kit b/apps/isaaclab.python.headless.rendering.kit index ddb7000b96c..836432cf647 100644 --- a/apps/isaaclab.python.headless.rendering.kit +++ b/apps/isaaclab.python.headless.rendering.kit @@ -18,6 +18,7 @@ keywords = ["experience", "app", "isaaclab", "python", "camera", "minimal"] # Isaac Lab minimal app "isaaclab.python.headless" = {} "omni.replicator.core" = {} +"isaacsim.sensors.rtx" = {} # Rendering "omni.kit.material.library" = {} @@ -86,6 +87,11 @@ app.vulkan = true # disable replicator orchestrator for better runtime perf exts."omni.replicator.core".Orchestrator.enabled = false +# move the default lidar config file paths +app.sensors.nv.lidar.profileBaseFolder.'++' = [ + "${app}/../source/isaaclab/isaaclab/sensors/rtx_lidar/", +] + [settings.exts."omni.kit.registry.nucleus"] registries = [ { name = "kit/default", url = "https://ovextensionsprod.blob.core.windows.net/exts/kit/prod/106/shared" }, diff --git a/apps/isaaclab.python.rendering.kit b/apps/isaaclab.python.rendering.kit index 1c52b38fd78..ab83ee29a78 100644 --- a/apps/isaaclab.python.rendering.kit +++ b/apps/isaaclab.python.rendering.kit @@ -17,6 +17,7 @@ keywords = ["experience", "app", "isaaclab", "python", "camera", "minimal"] [dependencies] # Isaac Lab minimal app "isaaclab.python" = {} +"isaacsim.sensors.rtx" = {} # PhysX "omni.kit.property.physx" = {} @@ -84,6 +85,11 @@ app.audio.enabled = false # disable replicator orchestrator for better runtime perf exts."omni.replicator.core".Orchestrator.enabled = false +# move the default lidar config file paths +app.sensors.nv.lidar.profileBaseFolder.'++' = [ + "${app}/../source/isaaclab/isaaclab/sensors/rtx_lidar/", +] + [settings.physics] updateToUsd = false updateParticlesToUsd = false diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/__init__.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/__init__.py new file mode 100644 index 00000000000..50697595980 --- /dev/null +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from .rtx_lidar import RtxLidar +from .rtx_lidar_cfg import RtxLidarCfg +from .rtx_lidar_data import RTX_LIDAR_INFO_FIELDS, RtxLidarData diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py new file mode 100644 index 00000000000..08fcdfd605c --- /dev/null +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py @@ -0,0 +1,342 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import os +import re +import torch +from collections.abc import Sequence +from typing import TYPE_CHECKING, Any + +import carb +import omni.kit.commands +import omni.usd +from isaacsim.core.prims import XFormPrim +from pxr import UsdGeom + +import isaaclab.sim as sim_utils +from isaaclab.utils.array import convert_to_torch + +from ..sensor_base import SensorBase +from .rtx_lidar_data import RTX_LIDAR_INFO_FIELDS, RtxLidarData + +if TYPE_CHECKING: + from .rtx_lidar_cfg import RtxLidarCfg + + +class RtxLidar(SensorBase): + """The RTX Lidar sensor to acquire render based lidar data. + + This class wraps over the `UsdGeom Camera`_ for providing a consistent API for acquiring LiDAR data. + + This implementation utilizes the "RtxSensorCpuIsaacCreateRTXLidarScanBuffer" annotator. + + RTX lidar: https://docs.omniverse.nvidia.com/isaacsim/latest/features/sensors_simulation/isaac_sim_sensors_rtx_based_lidar.html + LiDAR config files: https://docs.omniverse.nvidia.com/kit/docs/omni.sensors.nv.lidar/latest/lidar_extension.html + RTX lidar Annotators: https://docs.omniverse.nvidia.com/isaacsim/latest/features/sensors_simulation/isaac_sim_sensors_rtx_based_lidar/annotator_descriptions.html + """ + + cfg: RtxLidarCfg + + def __init__(self, cfg: RtxLidarCfg): + """Initializes the RTX lidar object. + + Args: + cfg: The configuration parameters. + + Raises: + RuntimeError: If provided path is a regex. + RuntimeError: If no camera prim is found at the given path. + """ + # check if sensor path is valid + # note: currently we do not handle environment indices if there is a regex pattern in the leaf + # For example, if the prim path is "/World/Sensor_[1,2]". + sensor_path = cfg.prim_path.split("/")[-1] + sensor_path_is_regex = re.match(r"^[a-zA-Z0-9/_]+$", sensor_path) is None + if sensor_path_is_regex: + raise RuntimeError( + f"Invalid prim path for the rtx lidar sensor: {self.cfg.prim_path}." + "\n\tHint: Please ensure that the prim path does not contain any regex patterns in the leaf." + ) + + # Initialize base class + super().__init__(cfg) + + # toggle rendering of rtx sensors as True + # this flag is read by SimulationContext to determine if rtx sensors should be rendered + carb_settings_iface = carb.settings.get_settings() + carb_settings_iface.set_bool("/isaaclab/render/rtx_sensors", True) + # spawn the asset + if self.cfg.spawn is not None: + self.cfg.spawn.func( + self.cfg.prim_path, self.cfg.spawn, translation=self.cfg.offset.pos, orientation=self.cfg.offset.rot + ) + # check that spawn was successful + matching_prims = sim_utils.find_matching_prims(self.cfg.prim_path) + if len(matching_prims) == 0: + raise RuntimeError(f"Could not find prim with path {self.cfg.prim_path}.") + + self._sensor_prims: list[UsdGeom.Camera] = list() + # Create empty variables for storing output data + self._data = RtxLidarData() + + def __del__(self): + """Unsubscribes from callbacks and detach from the replicator registry and clean up any custom lidar configs.""" + + # delete from replicator registry + for annotator, render_product_path in zip(self._rep_registry, self._render_product_paths): + annotator.detach([render_product_path]) + annotator = None + # delete custom lidar config temp files + if self.cfg.spawn.lidar_type == "Custom": + file_dir = self.cfg.spawn.sensor_profile_temp_dir + if os.path.isdir(file_dir): + for file in os.listdir(file_dir): + if self.cfg.spawn.sensor_profile_temp_prefix in file and os.path.isfile( + os.path.join(file_dir, file) + ): + os.remove(os.path.join(file_dir, file)) + + """ + Properties + """ + + @property + def num_instances(self) -> int: + return 1 + + @property + def data(self) -> RtxLidarData: + # update sensors if needed + self._update_outdated_buffers() + # return the data + return self._data + + @property + def frame(self) -> torch.tensor: + """Frame number when the measurement took place.""" + return self._frame + + @property + def render_product_paths(self) -> list[str]: + """The path of the render products for the cameras. + + This can be used via replicator interfaces to attach to writes or external annotator registry. + """ + return self._render_product_paths + + """ + Operations + """ + + def reset(self, env_ids: Sequence[int] | None = None): + if not self._is_initialized: + raise RuntimeError( + "Camera could not be initialized. Please ensure --enable_cameras is used to enable rendering." + ) + # reset the timestamps + super().reset(env_ids) + # resolve None + # note: cannot do smart indexing here since we do a for loop over data. + if env_ids is None: + env_ids = self._ALL_INDICES + # Reset the frame count + self._frame[env_ids] = 0 + + """ + Implementation. + """ + + def _initialize_impl(self): + """Initializes the sensor handles and internal buffers. + + This function creates handles and registers the optional output data fields with the replicator registry to + be able to access the data from the sensor. It also initializes the internal buffers to store the data. + + Raises: + RuntimeError: If the enable_camera flag is not set. + RuntimeError: If the number of camera prims in the view does not match the number of environments. + RuntimeError: If the provided USD prim is not a Camera. + """ + carb_settings_iface = carb.settings.get_settings() + if not carb_settings_iface.get("/isaaclab/cameras_enabled"): + raise RuntimeError( + "A camera was spawned without the --enable_cameras flag. Please use --enable_cameras to enable" + " rendering." + ) + + import omni.replicator.core as rep + + super()._initialize_impl() + # Create a view for the sensor + self._view = XFormPrim(self.cfg.prim_path, reset_xform_properties=False) + self._view.initialize() + # Check that sizes are correct + if self._view.count != self._num_envs: + raise RuntimeError( + f"Number of camera prims in the view ({self._view.count}) does not match" + f" the number of environments ({self._num_envs})." + ) + + # Create all env_ids buffer + self._ALL_INDICES = torch.arange(self._view.count, device=self._device, dtype=torch.long) + # Create frame count buffer + self._frame = torch.zeros(self._view.count, device=self._device, dtype=torch.long) + + # lidar_prim_paths = sim_utils.find_matching_prims(self.cfg.prim_path) + + self._render_product_paths: list[str] = list() + self._rep_registry: list[rep.annotators.Annotator] = [] + + # Obtain current stage + stage = omni.usd.get_context().get_stage() + + for lidar_prim_path in self._view.prim_paths: + # Get lidar prim + lidar_prim = stage.GetPrimAtPath(lidar_prim_path) + # Check if prim is a camera + if not lidar_prim.IsA(UsdGeom.Camera): + raise RuntimeError(f"Prim at path '{lidar_prim_path}' is not a Camera.") + # Add to list + sensor_prim = UsdGeom.Camera(lidar_prim) + self._sensor_prims.append(sensor_prim) + + init_params = { + "outputAzimuth": False, + "outputElevation": False, + "outputNormal": False, + "outputVelocity": False, + "outputBeamId": False, + "outputEmitterId": False, + "outputMaterialId": False, + "outputObjectId": False, + "outputTimestamp": True, # always turn on timestamp field + } + + # create annotator node + annotator_type = "RtxSensorCpuIsaacCreateRTXLidarScanBuffer" + rep_annotator = rep.AnnotatorRegistry.get_annotator(annotator_type) + # turn on any optional data type returns + for name in self.cfg.optional_data_types: + if name == "azimuth": + init_params["outputAzimuth"] = True + elif name == "elevation": + init_params["outputElevation"] = True + elif name == "normal": + init_params["outputNormal"] = True + elif name == "velocity": + init_params["outputVelocity"] = True + elif name == "beamId": + init_params["outputBeamId"] = True + elif name == "emitterId": + init_params["outputEmitterId"] = True + elif name == "materialId": + init_params["outputMaterialId"] = True + elif name == "objectId": + init_params["outputObjectId"] = True + + # transform the data output to be relative to sensor frame + if self.cfg.data_frame == "sensor": + init_params["transformPoints"] = False + + rep_annotator.initialize(**init_params) + + # Get render product + # From Isaac Sim 2023.1 onwards, render product is a HydraTexture so we need to extract the path + render_prod_path = rep.create.render_product(lidar_prim_path, [1, 1]) + if not isinstance(render_prod_path, str): + render_prod_path = render_prod_path.path + self._render_product_paths.append(render_prod_path) + + rep_annotator.attach(render_prod_path) + self._rep_registry.append(rep_annotator) + + # Debug draw + if self.cfg.debug_vis: + self.writer = rep.writers.get("RtxLidarDebugDrawPointCloudBuffer") + self.writer.attach([render_prod_path]) + + # Create internal buffers + self._create_buffers() + + def _create_buffers(self): + """Create buffers for storing data.""" + # create the data object + + # -- output data + # lazy allocation of data dictionary + # since the size of the output data is not known in advance, we leave it as None + # the memory will be allocated when the buffer() function is called for the first time. + self._data.output = {} + self._data.info = [{name: None for name in RTX_LIDAR_INFO_FIELDS.keys()} for _ in range(self._view.count)] + + def _update_buffers_impl(self, env_ids: Sequence[int]): + """Compute and fill sensor data buffers.""" + # Increment frame count + self._frame[env_ids] += 1 + data_all_lidar = list() + info_data_all_lidar: dict[str, list] = {} + + # iterate over all the annotators + for index in env_ids: + # get the output + output = self._rep_registry[index].get_data() + # process the output + data, info = self._process_annotator_output("", output) + + # add data to output + data_all_lidar.append(data) + + # store the info + for info_key, info_value in info.items(): + if info_key in RTX_LIDAR_INFO_FIELDS.keys(): + if info_key == "transform": + self._data.info[index][info_key] = torch.tensor(info_value, device=self.device) + else: + self._data.info[index][info_key] = info_value + else: + if info_key not in info_data_all_lidar: + info_data_all_lidar[info_key] = [torch.tensor(info_value, device=self._device)] + else: + info_data_all_lidar[info_key].append(torch.tensor(info_value, device=self._device)) + + # concatenate the data along the batch dimension + self._data.output["data"] = torch.stack(data_all_lidar, dim=0) + + for key in info_data_all_lidar: + self._data.output[key] = torch.stack(info_data_all_lidar[key], dim=0) + + def _process_annotator_output(self, name: str, output: Any) -> tuple[torch.tensor, dict | None]: + """Process the annotator output. + + This function is called after the data has been collected from all the cameras. + """ + # extract info and data from the output + if isinstance(output, dict): + data = output["data"] + info = output["info"] + else: + data = output + info = None + # convert data into torch tensor + data = convert_to_torch(data, device=self.device) + + # process data for different segmentation types + # Note: Replicator returns raw buffers of dtype int32 for segmentation types + # so we need to convert them to uint8 4 channel images for colorized types + + return data, info + + """ + Internal simulation callbacks. + """ + + def _invalidate_initialize_callback(self, event): + """Invalidates the scene elements.""" + # call parent + super()._invalidate_initialize_callback(event) + # set all existing views to None to invalidate them + self._view = None diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py new file mode 100644 index 00000000000..cefab0f133d --- /dev/null +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py @@ -0,0 +1,56 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +from dataclasses import MISSING +from typing import Literal + +from isaaclab.sensors import SensorBaseCfg +from isaaclab.sim import LidarCfg +from isaaclab.utils import configclass + +from .rtx_lidar import RtxLidar + + +@configclass +class RtxLidarCfg(SensorBaseCfg): + """Configuration for the RtxLidar sensor.""" + + @configclass + class OffsetCfg: + """The offset pose of the sensor's frame from the sensor's parent frame.""" + + pos: tuple[float, float, float] = (0.0, 0.0, 0.0) + """Translation w.r.t. the parent frame. Defaults to (0.0, 0.0, 0.0).""" + rot: tuple[float, float, float, float] = (1.0, 0.0, 0.0, 0.0) + """Quaternion rotation (w, x, y, z) w.r.t. the parent frame. Defaults to (1.0, 0.0, 0.0, 0.0).""" + + class_type: type = RtxLidar + + offset: OffsetCfg = OffsetCfg() + """The offset pose of the sensor's frame from the sensor's parent frame. Defaults to identity. + + Note: + The parent frame is the frame the sensor attaches to. For example, the parent frame of a + camera at path ``/World/envs/env_0/Robot/Camera`` is ``/World/envs/env_0/Robot``. + """ + optional_data_types: list[ + Literal["azimuth", "beamId", "elevation", "emitterId", "index", "materialId", "normal", "objectId", "velocity"] + ] = [] + """The optional output data types to include in RtxLidarData. + + Please refer to the :class:'RtxLidar' and :class:'RtxLidarData' for a list and description of available data types. + """ + data_frame: Literal["world", "sensor"] = "world" + """The frame to represent the output.data. + + If 'world' the output.data will be in the world frame. If 'sensor' the output.data will be in the sensor frame.""" + spawn: LidarCfg = MISSING + """Spawn configuration for the asset. + + If None, then the prim is not spawned by the asset. Instead, it is assumed that the + asset is already present in the scene. + """ diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_data.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_data.py new file mode 100644 index 00000000000..1102d5188b3 --- /dev/null +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_data.py @@ -0,0 +1,64 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +from __future__ import annotations + +import torch +from dataclasses import dataclass +from typing import Any + +RTX_LIDAR_INFO_FIELDS = { + "numChannels": int, + "numEchos": int, + "numReturnsPerScan": int, + "renderProductPath": str, + "ticksPerScan": int, + "transform": torch.Tensor, +} + + +@dataclass +class RtxLidarData: + """Data container for the RtxLidar sensor.""" + + info: list[dict[str, Any]] = None + """The static descriptive information from the lidar sensor. + + Each dictionary corresponds to an instance of a lidar and contains the following fields from the + "RtxSensorCpuIsaacCreateRTXLidarScanBuffer" annotator: + numChannels: The maximum number of channels. + numEchos: The maximum number echos. + numReturnsPerScan: The maximum number of returns possible in the output. + renderProductPath: The render product from the camera prim sensor. + ticksPerScan: The maximum number of ticks. + transform: Transform of the latest data added to the scan buffer for transforming the data to world space. + + The product of ticksPerScan, numChannels, and numEchos will be the same as the number of returns if you initialize + the annotator with annotator.initialize(keepOnlyPositiveDistance=False) before attaching the render product. + """ + output: dict[str, torch.Tensor] = None + """The data that changes every sample. Some fields of the out will always be returned and some are optionally + returned when configured in RtxLidarCfg.optional_data_types. + + The following keys will ALWAYS be returned: + data: The position [x,y,z] of each return in meter expressed in the RtxLidarCfg.data_frame. If 'world' the + data will be returned relative to simulation world. If 'sensor' the data will be returned relative to + sensor frame. + distance: The distance of the return hit from sensor origin in meters. + intensity: The intensity value in the range [0.0, 1.0] of each return. + timestamp: The time since sensor creation time in nanoseconds for each return. + + The following keys will OPTIONALLY be returned: + azimuth: The horizontal polar angle (radians) of the return. + elevation: the vertical polar angle (radians) of the return. + normal: The normal at the hit location in world coordinates. + velocity: The normalized velocity at the hit location in world coordinates. + emmitterId: Same as the channel unless the RTX Lidar Config Parameters is set up so emitters fire through + different channels. + materialId: The sensor material Id at the hit location. Same as index from rtSensorNameToIdMap setting in the RTX Sensor Visual Materials. + objectId: The object Id at the hit location. The objectId can be used to get the prim path of the object with the following code: + from omni.syntheticdata._syntheticdata import acquire_syntheticdata_interface + primpath = acquire_syntheticdata_interface().get_uri_from_instance_segmentation_id(object_id) + """ diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.py b/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.py index e2dd445788e..be24f75d8f4 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.py +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/__init__.py @@ -11,5 +11,5 @@ """ -from .sensors import spawn_camera -from .sensors_cfg import FisheyeCameraCfg, PinholeCameraCfg +from .sensors import spawn_camera, spawn_lidar +from .sensors_cfg import FisheyeCameraCfg, LidarCfg, PinholeCameraCfg diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py index b56134bfdc0..c9edd1fc122 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors.py @@ -5,12 +5,14 @@ from __future__ import annotations +import json +import os from typing import TYPE_CHECKING import isaacsim.core.utils.prims as prim_utils import omni.kit.commands import omni.log -from pxr import Sdf, Usd +from pxr import Gf, Sdf, Usd from isaaclab.sim.utils import clone from isaaclab.utils import to_camel_case @@ -140,3 +142,77 @@ def spawn_camera( prim.GetAttribute(prim_prop_name).Set(param_value) # return the prim return prim_utils.get_prim_at_path(prim_path) + + +@clone +def spawn_lidar( + prim_path: str, + cfg: sensors_cfg.LidarCfg, + translation: tuple[float, float, float] | None = None, + orientation: tuple[float, float, float, float] = (1.0, 0.0, 0.0, 0.0), +) -> Usd.Prim: + """Create a USD Camera prim used for RTX Lidar models. + This function creates an RTX lidar model attached to a USD Camera prim. The RTX lidar model is configured from json + files located within ``omni.isaac.core``. + Custom configurations for the RTX lidar are passed in via the :class:`LidarCfg`. By setting the `lidar_type` to + custom and providing a `sensor_profile` dictionary a json configuration file will be created and passed to the + kit commands responsible for the RTX lidar creation. + Args: + prim_path: The prim path or pattern to spawn the asset at. If the prim path is a regex pattern, + then the asset is spawned at all the matching prim paths. + cfg: The configuration instance. + translation: The translation to apply to the prim w.r.t. its parent prim. Defaults to None, in which case + this is set to the origin. + orientation: The orientation in (w, x, y, z) to apply to the prim w.r.t. its parent prim. Defaults to None, + in which case this is set to identity. + Returns: + The created prim. + Raises: + ValueError: If the LidarCfg.sensor_profile is None when LidarCfg.lidar_type == "Custom" + RuntimeError: If the creation of the RTXLidar fails + ValueError: If a prim already exists at the given path. + """ + + if cfg.lidar_type == "Custom": + if cfg.sensor_profile is None: + raise ValueError("LidarCfg sensor_profile cannot be none for lidar_type: Custom") + + # make directories + if not os.path.isdir(cfg.sensor_profile_temp_dir): + os.makedirs(cfg.sensor_profile_temp_dir) + + # create file path + file_name = cfg.sensor_profile_temp_prefix + ".json" + file_path = os.path.join(cfg.sensor_profile_temp_dir, file_name) + + # Check for tempfiles and remove + while os.path.isfile(file_path): + os.remove(file_path) + + # Write to file + with open(file_path, "w") as outfile: + json.dump(cfg.sensor_profile, outfile) + + print("Custom: ") + config = file_path.split("/")[-1].split(".")[0] + print(file_path) + else: + config = cfg.lidar_type + + if not prim_utils.is_prim_path_valid(prim_path): + # prim_utils.create_prim(prim_path, "Camera", translation=translation, orientation=orientation) + _, sensor = omni.kit.commands.execute( + "IsaacSensorCreateRtxLidar", + path=prim_path, + parent=None, + config=config, + translation=translation, + orientation=Gf.Quatd(orientation[0], orientation[1], orientation[2], orientation[3]), + ) + if not sensor.IsValid(): + raise RuntimeError("IsaacSensorCreateRtxLidar failed to create a USD.Prim") + + else: + raise ValueError(f"A prim already exists at path: '{prim_path}'.") + + return sensor diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py index cff5f9be17f..5f247076dab 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py @@ -5,9 +5,11 @@ from __future__ import annotations +import os from collections.abc import Callable -from typing import Literal +from typing import Any, Literal +from isaaclab import ISAACLAB_EXT_DIR from isaaclab.sim.spawners.spawner_cfg import SpawnerCfg from isaaclab.utils import configclass @@ -233,3 +235,81 @@ class FisheyeCameraCfg(PinholeCameraCfg): fisheye_polynomial_f: float = 0.0 """Sixth component of fisheye polynomial. Defaults to 0.0.""" + + +@configclass +class LidarCfg(SpawnerCfg): + """ + Lidar Configuration Table + +-------------+-----------+-----------------------------------+---------------------------+ + | Manufacturer| Model | UI Name | Config Name | + +=============+===========+===================================+===========================+ + | HESAI |PandarXT-32| PandarXT-32 10hz | Hesai_XT32_SD10 | + +-------------+-----------+-----------------------------------+---------------------------+ + | Ouster | OS0 | OS0 128 10hz @ 1024 resolution | OS0_128ch10hz1024res | + | | | OS0 128 10hz @ 2048 resolution | OS0_128ch10hz2048res | + | | | OS0 128 10hz @ 512 resolution | OS0_128ch10hz512res | + | | | OS0 128 20hz @ 1024 resolution | OS0_128ch20hz1024res | + | | | OS0 128 20hz @ 512 resolution | OS0_128ch20hz512res | + +-------------+-----------+-----------------------------------+---------------------------+ + | Ouster | OS1 | OS1 32 10hz @ 1024 resolution | OS1_32ch10hz1024res | + | | | OS1 32 10hz @ 2048 resolution | OS1_32ch10hz2048res | + | | | OS1 32 10hz @ 512 resolution | OS1_32ch10hz512res | + | | | OS1 32 20hz @ 1024 resolution | OS1_32ch20hz1024res | + | | | OS1 32 20hz @ 512 resolution | OS1_32ch20hz512res | + +-------------+-----------+-----------------------------------+---------------------------+ + | SICK | TiM781 | SICK TiM781 | Sick_TiM781 | + +-------------+-----------+-----------------------------------+---------------------------+ + | SLAMTEC |RPLidar S2E| RPLidar S2E | RPLIDAR_S2E | + +-------------+-----------+-----------------------------------+---------------------------+ + | Velodyne | VLS-128 | Velodyne VLS-128 | Velodyne_VLS128 | + +-------------+-----------+-----------------------------------+---------------------------+ + | ZVISION | ML-30s+ | ML-30s+ | ZVISION_ML30S | + | | ML-Xs | ML-Xs | ZVISION_MLXS | + +-------------+-----------+-----------------------------------+---------------------------+ + | NVIDIA | Generic | Rotating | Example_Rotary | + | | Generic | Solid State | Example_Solid_State | + | | Debug | Simple Solid State | Simple_Example_Solid_State| + +-------------+-----------+-----------------------------------+---------------------------+ + """ + + func = sensors.spawn_lidar + """The RTX lidar spawn function.""" + + lidar_type: str = "Example_Rotary" + """The name of the lidar sensor profile. Defaults to Example_Rotatry. + There are many built in configuration files specified by LidarType below. + If a user want to create a custom configuration file set lidar_type="Custom" and create a sensor_profile dictionary.""" + + class LidarType: + """Class variables for autocompletion""" + + HESAI_PandarXT_32 = "Hesai_XT32_SD10" + OUSTER_OS0_128_10HZ_1024RES = "OS0_128ch10hz1024res" + OUSTER_OS0_128_10HZ_2048RES = "OS0_128ch10hz2048res" + OUSTER_OS0_128_10HZ_512RES = "OS0_128ch10hz512res" + OUSTER_OS0_128_20HZ_1024RES = "OS0_128ch20hz1024res" + OUSTER_OS0_128_20HZ_512RES = "OS0_128ch20hz512res" + OUSTER_OS1_32_10HZ_1024RES = "OS1_32ch10hz1024res" + OUSTER_OS1_32_10HZ_2048RES = "OS1_32ch10hz2048res" + OUSTER_OS1_32_10HZ_512RES = "OS1_32ch10hz512res" + OUSTER_OS1_32_20HZ_1024RES = "OS1_32ch20hz1024res" + OUSTER_OS1_32_20HZ_512RES = "OS1_32ch20hz512res" + SICK_TIM781 = "Sick_TiM781" + SLAMTEC_RPLIDAR_S2E = "RPLIDAR_S2E" + VELODYNE_VLS128 = "Velodyne_VLS128" + ZVISION_ML30S = "ZVISION_ML30S" + ZVISION_MLXS = "ZVISION_MLXS" + EXAMPLE_ROTARY = "Example_Rotary" + EXAMPLE_SOLID_STATE = "Example_Solid_State" + SIMPLE_EXAMPLE_SOLID_STATE = "Simple_Example_Solid_State" + + sensor_profile: dict[str, Any] | None = None + """Custom lidar parameters to use if lidar_type="Custom" + see https://docs.omniverse.nvidia.com/kit/docs/omni.sensors.nv.lidar/latest/lidar_extension.html""" + + sensor_profile_temp_dir: str = os.path.abspath(os.path.join(ISAACLAB_EXT_DIR, "isaaclab/sensors/rtx_lidar")) + """The location of the generated custom sensor profile json file.""" + + sensor_profile_temp_prefix: str = "Temp_Config_" + """The custom sensor profile json file prefix. This is used for cleanup of the custom sensor profile.""" diff --git a/source/isaaclab/test/sensors/test_rtx_lidar.py b/source/isaaclab/test/sensors/test_rtx_lidar.py new file mode 100644 index 00000000000..d9de55a2143 --- /dev/null +++ b/source/isaaclab/test/sensors/test_rtx_lidar.py @@ -0,0 +1,260 @@ +# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# All rights reserved. +# +# SPDX-License-Identifier: BSD-3-Clause + +# ignore private usage of variables warning +# pyright: reportPrivateUsage=none + +"""Launch Isaac Sim Simulator first.""" + +from isaaclab.app import AppLauncher, run_tests + +# launch omniverse app +app_launcher = AppLauncher(headless=True, enable_cameras=True) +simulation_app = app_launcher.app + +import copy +import json +import numpy as np +import os +import scipy.spatial.transform as tf +import torch +import unittest + +import isaacsim.core.utils.stage as stage_utils +import omni.replicator.core as rep +from isaacsim.core.utils.extensions import get_extension_path_from_name +from pxr import Usd, UsdGeom + +import isaaclab.sim as sim_utils +from isaaclab.sensors.rtx_lidar import RTX_LIDAR_INFO_FIELDS, RtxLidar, RtxLidarCfg +from isaaclab.terrains.trimesh.utils import make_border, make_plane +from isaaclab.terrains.utils import create_prim_from_mesh +from isaaclab.utils.math import convert_quat + +POSITION = (0.0, 0.0, 0.5) +QUATERNION = (0.0, 0.3461835, 0.0, 0.9381668) + +# load example json +EXAMPLE_ROTARY_PATH = os.path.abspath( + os.path.join( + get_extension_path_from_name("isaacsim.sensors.rtx"), + "data/lidar_configs/NVIDIA/Simple_Example_Solid_State.json", + ) +) + + +class TestRtxLidar(unittest.TestCase): + """Test for isaaclab rtx lidar""" + + """ + Test Setup and Teardown + """ + + def setUp(self): + """Create a blank new stage for each test.""" + + # Create a new stage + stage_utils.create_new_stage() + + # Simulation time-step + self.dt = 0.01 + # Load kit helper + sim_cfg = sim_utils.SimulationCfg(dt=self.dt, device="cuda") + self.sim: sim_utils.SimulationContext = sim_utils.SimulationContext(sim_cfg) + + # configure lidar + self.lidar_cfg = RtxLidarCfg( + prim_path="/World/Lidar", + debug_vis=not app_launcher._headless, + optional_data_types=[ + "azimuth", + "elevation", + "emitterId", + "index", + "materialId", + "normal", + "objectId", + "velocity", + ], + spawn=sim_utils.LidarCfg(lidar_type=sim_utils.LidarCfg.LidarType.EXAMPLE_ROTARY), + ) + + # Ground-plane + mesh = make_plane(size=(10, 10), height=0.0, center_zero=True) + border = make_border(size=(10, 10), inner_size=(5, 5), height=2.0, position=(0.0, 0.0, 0.0)) + + create_prim_from_mesh("/World/defaultGroundPlane", mesh) + for i, box in enumerate(border): + create_prim_from_mesh(f"/World/defaultBoarder{i}", box) + # load stage + stage_utils.update_stage() + + def tearDown(self): + """Stops simulator after each test.""" + # close all the opened viewport from before. + rep.vp_manager.destroy_hydra_textures("Replicator") + # stop simulation + # note: cannot use self.sim.stop() since it does one render step after stopping!! This doesn't make sense :( + self.sim._timeline.stop() + # clear the stage + self.sim.clear_all_callbacks() + self.sim.clear_instance() + + def test_lidar_init(self): + """Test lidar initialization and data population.""" + # Create lidar + lidar = RtxLidar(cfg=self.lidar_cfg) + # Check simulation parameter is set correctly + self.assertTrue(self.sim.has_rtx_sensors()) + # Play sim + self.sim.reset() + # Check if lidar is initialized + self.assertTrue(lidar.is_initialized) + # Check if lidar prim is set correctly and that it is a camera prim + self.assertEqual(lidar._sensor_prims[0].GetPath().pathString, self.lidar_cfg.prim_path) + self.assertIsInstance(lidar._sensor_prims[0], UsdGeom.Camera) + + # Simulate for a few steps + # note: This is a workaround to ensure that the textures are loaded. + # Check "Known Issues" section in the documentation for more details. + for _ in range(5): + self.sim.step() + + # Simulate physics + for _ in range(10): + # perform rendering + self.sim.step() + # update camera + lidar.update(self.dt, force_recompute=True) + # check info data + for info_key, info_value in lidar.data.info[0].items(): + self.assertTrue(info_key in RTX_LIDAR_INFO_FIELDS.keys()) + self.assertTrue(isinstance(info_value, RTX_LIDAR_INFO_FIELDS[info_key])) + + # check lidar data + for data_key, data_value in lidar.data.output.items(): + if data_key in self.lidar_cfg.optional_data_types: + self.assertTrue(data_value.shape[1] > 0) + + def test_lidar_init_offset(self): + """Test lidar offset configuration.""" + lidar_cfg_offset = copy.deepcopy(self.lidar_cfg) + lidar_cfg_offset.offset = RtxLidarCfg.OffsetCfg(pos=POSITION, rot=QUATERNION) + lidar_cfg_offset.prim_path = "/World/LidarOffset" + lidar_offset = RtxLidar(lidar_cfg_offset) + # Play sim + self.sim.reset() + + # Retrieve lidar pose using USD API + prim_tf = lidar_offset._sensor_prims[0].ComputeLocalToWorldTransform(Usd.TimeCode.Default()) + prim_tf = np.transpose(prim_tf) + + # check that transform is set correctly + np.testing.assert_allclose(prim_tf[0:3, 3], lidar_cfg_offset.offset.pos) + np.testing.assert_allclose( + convert_quat(tf.Rotation.from_matrix(prim_tf[:3, :3]).as_quat(), "wxyz"), + lidar_cfg_offset.offset.rot, + rtol=1e-5, + atol=1e-5, + ) + + def test_multi_lidar_init(self): + """Test multiple lidar initialization and check info and data outputs.""" + lidar_cfg_1 = copy.deepcopy(self.lidar_cfg) + lidar_cfg_1.prim_path = "/World/Lidar1" + lidar_1 = RtxLidar(lidar_cfg_1) + + lidar_cfg_2 = copy.deepcopy(self.lidar_cfg) + lidar_cfg_2.prim_path = "/World/Lidar2" + lidar_2 = RtxLidar(lidar_cfg_2) + + # play sim + self.sim.reset() + + # Simulate for a few steps + # note: This is a workaround to ensure that the textures are loaded. + # Check "Known Issues" section in the documentation for more details. + for _ in range(5): + self.sim.step() + # Simulate physics + for i in range(10): + # perform rendering + self.sim.step() + # update lidar + lidar_1.update(self.dt, force_recompute=True) + lidar_2.update(self.dt, force_recompute=True) + # check lidar info + for lidar_info_key in lidar_1.data.info[0].keys(): + info1 = lidar_1.data.info[0][lidar_info_key] + info2 = lidar_2.data.info[0][lidar_info_key] + if isinstance(info1, torch.Tensor): + torch.testing.assert_close(info1, info2) + else: + if lidar_info_key == "renderProductPath": + self.assertTrue(info1 == info2.split("_")[0]) + else: + self.assertTrue(info1 == info2) + # check lidar data shape + for lidar_data_key in lidar_1.data.output.keys(): + data1 = lidar_1.data.output[lidar_data_key] + data2 = lidar_2.data.output[lidar_data_key] + self.assertTrue(data1.shape == data2.shape) + + def test_custom_lidar_config(self): + """Test custom lidar initialization, data population, and cleanup.""" + # Create custom lidar profile dictionary + with open(EXAMPLE_ROTARY_PATH) as json_file: + sensor_profile = json.load(json_file) + + custom_lidar_cfg = copy.deepcopy(self.lidar_cfg) + custom_lidar_cfg.spawn = sim_utils.LidarCfg(lidar_type="Custom", sensor_profile=sensor_profile) + # Create custom lidar + lidar = RtxLidar(cfg=custom_lidar_cfg) + # Check simulation parameter is set correctly + self.assertTrue(self.sim.has_rtx_sensors()) + # Play sim + self.sim.reset() + # Check if lidar is initialized + self.assertTrue(lidar.is_initialized) + # Check if lidar prim is set correctly and that it is a camera prim + self.assertEqual(lidar._sensor_prims[0].GetPath().pathString, self.lidar_cfg.prim_path) + self.assertIsInstance(lidar._sensor_prims[0], UsdGeom.Camera) + + # Simulate for a few steps + # note: This is a workaround to ensure that the textures are loaded. + # Check "Known Issues" section in the documentation for more details. + for _ in range(5): + self.sim.step() + + # Simulate physics + for _ in range(10): + # perform rendering + self.sim.step() + # update camera + lidar.update(self.dt, force_recompute=True) + # check info data + for info_key, info_value in lidar.data.info[0].items(): + self.assertTrue(info_key in RTX_LIDAR_INFO_FIELDS.keys()) + self.assertTrue(isinstance(info_value, RTX_LIDAR_INFO_FIELDS[info_key])) + + # check lidar data + for data_key, data_value in lidar.data.output.items(): + if data_key in self.lidar_cfg.optional_data_types: + self.assertTrue(data_value.shape[1] > 0) + + del lidar + + # check proper file cleanup + custom_profile_name = self.lidar_cfg.spawn.sensor_profile_temp_prefix + custom_profile_dir = self.lidar_cfg.spawn.sensor_profile_temp_dir + files = os.listdir(custom_profile_dir) + for file in files: + self.assertTrue( + custom_profile_name not in file, msg=f"{custom_profile_name} found in {custom_profile_dir}/{file}" + ) + + +if __name__ == "__main__": + run_tests() From 413c7fadf27b30412b89cd0905b5bbcefbf1cc2e Mon Sep 17 00:00:00 2001 From: James Tigue Date: Mon, 14 Apr 2025 16:27:26 -0400 Subject: [PATCH 02/12] fixing tests --- .../isaaclab/sensors/rtx_lidar/rtx_lidar.py | 9 +++--- .../sensors/rtx_lidar/rtx_lidar_cfg.py | 3 +- .../isaaclab/test/sensors/test_rtx_lidar.py | 32 ++++++++++++++----- 3 files changed, 31 insertions(+), 13 deletions(-) diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py index 08fcdfd605c..effe22c4811 100644 --- a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py @@ -34,9 +34,9 @@ class RtxLidar(SensorBase): This implementation utilizes the "RtxSensorCpuIsaacCreateRTXLidarScanBuffer" annotator. - RTX lidar: https://docs.omniverse.nvidia.com/isaacsim/latest/features/sensors_simulation/isaac_sim_sensors_rtx_based_lidar.html - LiDAR config files: https://docs.omniverse.nvidia.com/kit/docs/omni.sensors.nv.lidar/latest/lidar_extension.html - RTX lidar Annotators: https://docs.omniverse.nvidia.com/isaacsim/latest/features/sensors_simulation/isaac_sim_sensors_rtx_based_lidar/annotator_descriptions.html + .. RTX lidar: + .. LiDAR config files: + .. RTX lidar Annotators: """ cfg: RtxLidarCfg @@ -85,7 +85,8 @@ def __init__(self, cfg: RtxLidarCfg): def __del__(self): """Unsubscribes from callbacks and detach from the replicator registry and clean up any custom lidar configs.""" - + # unsubscribe callbacks + super().__del__() # delete from replicator registry for annotator, render_product_path in zip(self._rep_registry, self._render_product_paths): annotator.detach([render_product_path]) diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py index cefab0f133d..236b3c00047 100644 --- a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py @@ -47,7 +47,8 @@ class OffsetCfg: data_frame: Literal["world", "sensor"] = "world" """The frame to represent the output.data. - If 'world' the output.data will be in the world frame. If 'sensor' the output.data will be in the sensor frame.""" + If 'world' the output.data will be in the world frame. If 'sensor' the output.data will be in the sensor frame. + """ spawn: LidarCfg = MISSING """Spawn configuration for the asset. diff --git a/source/isaaclab/test/sensors/test_rtx_lidar.py b/source/isaaclab/test/sensors/test_rtx_lidar.py index d9de55a2143..14ab882e541 100644 --- a/source/isaaclab/test/sensors/test_rtx_lidar.py +++ b/source/isaaclab/test/sensors/test_rtx_lidar.py @@ -11,7 +11,7 @@ from isaaclab.app import AppLauncher, run_tests # launch omniverse app -app_launcher = AppLauncher(headless=True, enable_cameras=True) +app_launcher = AppLauncher(headless=False, enable_cameras=True) simulation_app = app_launcher.app import copy @@ -33,8 +33,9 @@ from isaaclab.terrains.utils import create_prim_from_mesh from isaaclab.utils.math import convert_quat -POSITION = (0.0, 0.0, 0.5) +POSITION = (0.0, 0.0, 0.1) QUATERNION = (0.0, 0.3461835, 0.0, 0.9381668) +# QUATERNION = (0.0, 0.0,0.0,1.0) # load example json EXAMPLE_ROTARY_PATH = os.path.abspath( @@ -61,7 +62,7 @@ def setUp(self): # Simulation time-step self.dt = 0.01 # Load kit helper - sim_cfg = sim_utils.SimulationCfg(dt=self.dt, device="cuda") + sim_cfg = sim_utils.SimulationCfg(dt=self.dt, device="cpu") self.sim: sim_utils.SimulationContext = sim_utils.SimulationContext(sim_cfg) # configure lidar @@ -137,18 +138,20 @@ def test_lidar_init(self): for data_key, data_value in lidar.data.output.items(): if data_key in self.lidar_cfg.optional_data_types: self.assertTrue(data_value.shape[1] > 0) + del lidar def test_lidar_init_offset(self): """Test lidar offset configuration.""" lidar_cfg_offset = copy.deepcopy(self.lidar_cfg) lidar_cfg_offset.offset = RtxLidarCfg.OffsetCfg(pos=POSITION, rot=QUATERNION) lidar_cfg_offset.prim_path = "/World/LidarOffset" - lidar_offset = RtxLidar(lidar_cfg_offset) + lidar = RtxLidar(lidar_cfg_offset) + # Play sim self.sim.reset() # Retrieve lidar pose using USD API - prim_tf = lidar_offset._sensor_prims[0].ComputeLocalToWorldTransform(Usd.TimeCode.Default()) + prim_tf = lidar._sensor_prims[0].ComputeLocalToWorldTransform(Usd.TimeCode.Default()) prim_tf = np.transpose(prim_tf) # check that transform is set correctly @@ -160,6 +163,14 @@ def test_lidar_init_offset(self): atol=1e-5, ) + # Simulate for a few steps + # note: This is a workaround to ensure that the textures are loaded. + # Check "Known Issues" section in the documentation for more details. + for _ in range(5): + self.sim.step() + + del lidar + def test_multi_lidar_init(self): """Test multiple lidar initialization and check info and data outputs.""" lidar_cfg_1 = copy.deepcopy(self.lidar_cfg) @@ -183,8 +194,8 @@ def test_multi_lidar_init(self): # perform rendering self.sim.step() # update lidar - lidar_1.update(self.dt, force_recompute=True) - lidar_2.update(self.dt, force_recompute=True) + lidar_1.update(self.dt) + lidar_2.update(self.dt) # check lidar info for lidar_info_key in lidar_1.data.info[0].keys(): info1 = lidar_1.data.info[0][lidar_info_key] @@ -200,7 +211,12 @@ def test_multi_lidar_init(self): for lidar_data_key in lidar_1.data.output.keys(): data1 = lidar_1.data.output[lidar_data_key] data2 = lidar_2.data.output[lidar_data_key] - self.assertTrue(data1.shape == data2.shape) + self.assertTrue( + data1.shape == data2.shape, f"Key: {lidar_data_key}, Shape 1: {data1.shape}, Shape 2: {data2.shape}" + ) + + del lidar_1 + del lidar_2 def test_custom_lidar_config(self): """Test custom lidar initialization, data population, and cleanup.""" From 2c579b3227f89a28e569145432a4d29e4d8bf98c Mon Sep 17 00:00:00 2001 From: James Tigue Date: Mon, 14 Apr 2025 17:07:09 -0400 Subject: [PATCH 03/12] update doc refs --- source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py index effe22c4811..2100eaef757 100644 --- a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py @@ -34,9 +34,8 @@ class RtxLidar(SensorBase): This implementation utilizes the "RtxSensorCpuIsaacCreateRTXLidarScanBuffer" annotator. - .. RTX lidar: - .. LiDAR config files: - .. RTX lidar Annotators: + .. RTX lidar: + .. RTX lidar Annotators: """ cfg: RtxLidarCfg From c33dd8baf8988285bd800afd1d72f053293d0983 Mon Sep 17 00:00:00 2001 From: James Tigue Date: Tue, 15 Apr 2025 15:33:15 -0400 Subject: [PATCH 04/12] test fixes --- .../isaaclab/sensors/rtx_lidar/rtx_lidar.py | 2 +- .../isaaclab/test/sensors/test_rtx_lidar.py | 21 ++++++++++--------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py index 2100eaef757..76d3c628a3a 100644 --- a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py @@ -106,7 +106,7 @@ def __del__(self): @property def num_instances(self) -> int: - return 1 + return self._view.count @property def data(self) -> RtxLidarData: diff --git a/source/isaaclab/test/sensors/test_rtx_lidar.py b/source/isaaclab/test/sensors/test_rtx_lidar.py index 14ab882e541..951869dd90d 100644 --- a/source/isaaclab/test/sensors/test_rtx_lidar.py +++ b/source/isaaclab/test/sensors/test_rtx_lidar.py @@ -11,7 +11,7 @@ from isaaclab.app import AppLauncher, run_tests # launch omniverse app -app_launcher = AppLauncher(headless=False, enable_cameras=True) +app_launcher = AppLauncher(headless=True, enable_cameras=True) simulation_app = app_launcher.app import copy @@ -33,7 +33,7 @@ from isaaclab.terrains.utils import create_prim_from_mesh from isaaclab.utils.math import convert_quat -POSITION = (0.0, 0.0, 0.1) +POSITION = (0.0, 0.0, 0.3) QUATERNION = (0.0, 0.3461835, 0.0, 0.9381668) # QUATERNION = (0.0, 0.0,0.0,1.0) @@ -62,7 +62,7 @@ def setUp(self): # Simulation time-step self.dt = 0.01 # Load kit helper - sim_cfg = sim_utils.SimulationCfg(dt=self.dt, device="cpu") + sim_cfg = sim_utils.SimulationCfg(dt=self.dt, device="cuda") self.sim: sim_utils.SimulationContext = sim_utils.SimulationContext(sim_cfg) # configure lidar @@ -138,7 +138,6 @@ def test_lidar_init(self): for data_key, data_value in lidar.data.output.items(): if data_key in self.lidar_cfg.optional_data_types: self.assertTrue(data_value.shape[1] > 0) - del lidar def test_lidar_init_offset(self): """Test lidar offset configuration.""" @@ -169,10 +168,12 @@ def test_lidar_init_offset(self): for _ in range(5): self.sim.step() - del lidar + lidar.update(self.dt) - def test_multi_lidar_init(self): + def test_lidar_init_multiple(self): """Test multiple lidar initialization and check info and data outputs.""" + + self.sim._app_control_on_stop_handle = None lidar_cfg_1 = copy.deepcopy(self.lidar_cfg) lidar_cfg_1.prim_path = "/World/Lidar1" lidar_1 = RtxLidar(lidar_cfg_1) @@ -211,14 +212,14 @@ def test_multi_lidar_init(self): for lidar_data_key in lidar_1.data.output.keys(): data1 = lidar_1.data.output[lidar_data_key] data2 = lidar_2.data.output[lidar_data_key] + # print(i,"Frame1: ",lidar_1.frame) + # print(i,"Frame2: ",lidar_2.frame) + # print(i,f"Key: {lidar_data_key}, Shape 1: {data1.shape}, Shape 2: {data2.shape}") self.assertTrue( data1.shape == data2.shape, f"Key: {lidar_data_key}, Shape 1: {data1.shape}, Shape 2: {data2.shape}" ) - del lidar_1 - del lidar_2 - - def test_custom_lidar_config(self): + def test_lidar_init_custom(self): """Test custom lidar initialization, data population, and cleanup.""" # Create custom lidar profile dictionary with open(EXAMPLE_ROTARY_PATH) as json_file: From bb853ee52fc73a280dc9f282741d0da9d646a267 Mon Sep 17 00:00:00 2001 From: James Tigue Date: Wed, 16 Apr 2025 14:32:32 -0400 Subject: [PATCH 05/12] switch to pytest --- source/isaaclab/setup.py | 3 + .../isaaclab/test/sensors/test_rtx_lidar.py | 417 +++++++++--------- 2 files changed, 202 insertions(+), 218 deletions(-) diff --git a/source/isaaclab/setup.py b/source/isaaclab/setup.py index 86e9e4e88cb..e217ff0a0e0 100644 --- a/source/isaaclab/setup.py +++ b/source/isaaclab/setup.py @@ -38,6 +38,9 @@ "pillow==11.0.0", # livestream "starlette==0.46.0", + # testing + "pytest", + "pytest-mock", ] PYTORCH_INDEX_URL = ["https://download.pytorch.org/whl/cu118"] diff --git a/source/isaaclab/test/sensors/test_rtx_lidar.py b/source/isaaclab/test/sensors/test_rtx_lidar.py index 951869dd90d..51e84d4b020 100644 --- a/source/isaaclab/test/sensors/test_rtx_lidar.py +++ b/source/isaaclab/test/sensors/test_rtx_lidar.py @@ -8,7 +8,7 @@ """Launch Isaac Sim Simulator first.""" -from isaaclab.app import AppLauncher, run_tests +from isaaclab.app import AppLauncher # launch omniverse app app_launcher = AppLauncher(headless=True, enable_cameras=True) @@ -20,15 +20,15 @@ import os import scipy.spatial.transform as tf import torch -import unittest import isaacsim.core.utils.stage as stage_utils -import omni.replicator.core as rep +import pytest from isaacsim.core.utils.extensions import get_extension_path_from_name from pxr import Usd, UsdGeom import isaaclab.sim as sim_utils from isaaclab.sensors.rtx_lidar import RTX_LIDAR_INFO_FIELDS, RtxLidar, RtxLidarCfg +from isaaclab.sim import build_simulation_context from isaaclab.terrains.trimesh.utils import make_border, make_plane from isaaclab.terrains.utils import create_prim_from_mesh from isaaclab.utils.math import convert_quat @@ -46,42 +46,12 @@ ) -class TestRtxLidar(unittest.TestCase): - """Test for isaaclab rtx lidar""" - - """ - Test Setup and Teardown - """ - - def setUp(self): - """Create a blank new stage for each test.""" - - # Create a new stage - stage_utils.create_new_stage() - - # Simulation time-step - self.dt = 0.01 - # Load kit helper - sim_cfg = sim_utils.SimulationCfg(dt=self.dt, device="cuda") - self.sim: sim_utils.SimulationContext = sim_utils.SimulationContext(sim_cfg) - - # configure lidar - self.lidar_cfg = RtxLidarCfg( - prim_path="/World/Lidar", - debug_vis=not app_launcher._headless, - optional_data_types=[ - "azimuth", - "elevation", - "emitterId", - "index", - "materialId", - "normal", - "objectId", - "velocity", - ], - spawn=sim_utils.LidarCfg(lidar_type=sim_utils.LidarCfg.LidarType.EXAMPLE_ROTARY), - ) - +@pytest.fixture +def sim(request): + """Create simulation context with the specified device.""" + device = request.getfixturevalue("device") + with build_simulation_context(device=device, dt=0.01) as sim: + sim._app_control_on_stop_handle = None # Ground-plane mesh = make_plane(size=(10, 10), height=0.0, center_zero=True) border = make_border(size=(10, 10), inner_size=(5, 5), height=2.0, position=(0.0, 0.0, 0.0)) @@ -91,187 +61,198 @@ def setUp(self): create_prim_from_mesh(f"/World/defaultBoarder{i}", box) # load stage stage_utils.update_stage() + yield sim + + +@pytest.fixture +def lidar_cfg(request): + # configure lidar + return RtxLidarCfg( + prim_path="/World/Lidar", + debug_vis=not app_launcher._headless, + optional_data_types=[ + "azimuth", + "elevation", + "emitterId", + "index", + "materialId", + "normal", + "objectId", + "velocity", + ], + spawn=sim_utils.LidarCfg(lidar_type=sim_utils.LidarCfg.LidarType.EXAMPLE_ROTARY), + ) + + +@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +def test_lidar_init(sim, device, lidar_cfg): + """Test lidar initialization and data population.""" + # Create lidar + lidar = RtxLidar(cfg=lidar_cfg) + # Check simulation parameter is set correctly + assert sim.has_rtx_sensors() + # Play sim + sim.reset() + # Check if lidar is initialized + assert lidar.is_initialized + # Check if lidar prim is set correctly and that it is a camera prim + assert lidar._sensor_prims[0].GetPath().pathString == lidar_cfg.prim_path + assert isinstance(lidar._sensor_prims[0], UsdGeom.Camera) + + # Simulate for a few steps + # note: This is a workaround to ensure that the textures are loaded. + # Check "Known Issues" section in the documentation for more details. + for _ in range(5): + sim.step() + + # Simulate physics + for _ in range(10): + # perform rendering + sim.step() + # update camera + lidar.update(sim.get_physics_dt(), force_recompute=True) + # check info data + for info_key, info_value in lidar.data.info[0].items(): + assert info_key in RTX_LIDAR_INFO_FIELDS.keys() + assert isinstance(info_value, RTX_LIDAR_INFO_FIELDS[info_key]) + + # check lidar data + for data_key, data_value in lidar.data.output.items(): + if data_key in lidar_cfg.optional_data_types: + assert data_value.shape[1] > 0 + + +@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +def test_offset_lidar_init(sim, device, lidar_cfg): + """Test lidar offset configuration.""" + lidar_cfg_offset = copy.deepcopy(lidar_cfg) + lidar_cfg_offset.offset = RtxLidarCfg.OffsetCfg(pos=POSITION, rot=QUATERNION) + lidar_cfg_offset.prim_path = "/World/LidarOffset" + lidar = RtxLidar(lidar_cfg_offset) + + # Play sim + sim.reset() + + # Retrieve lidar pose using USD API + prim_tf = lidar._sensor_prims[0].ComputeLocalToWorldTransform(Usd.TimeCode.Default()) + prim_tf = np.transpose(prim_tf) + + # check that transform is set correctly + np.testing.assert_allclose(prim_tf[0:3, 3], lidar_cfg_offset.offset.pos) + np.testing.assert_allclose( + convert_quat(tf.Rotation.from_matrix(prim_tf[:3, :3]).as_quat(), "wxyz"), + lidar_cfg_offset.offset.rot, + rtol=1e-5, + atol=1e-5, + ) - def tearDown(self): - """Stops simulator after each test.""" - # close all the opened viewport from before. - rep.vp_manager.destroy_hydra_textures("Replicator") - # stop simulation - # note: cannot use self.sim.stop() since it does one render step after stopping!! This doesn't make sense :( - self.sim._timeline.stop() - # clear the stage - self.sim.clear_all_callbacks() - self.sim.clear_instance() - - def test_lidar_init(self): - """Test lidar initialization and data population.""" - # Create lidar - lidar = RtxLidar(cfg=self.lidar_cfg) - # Check simulation parameter is set correctly - self.assertTrue(self.sim.has_rtx_sensors()) - # Play sim - self.sim.reset() - # Check if lidar is initialized - self.assertTrue(lidar.is_initialized) - # Check if lidar prim is set correctly and that it is a camera prim - self.assertEqual(lidar._sensor_prims[0].GetPath().pathString, self.lidar_cfg.prim_path) - self.assertIsInstance(lidar._sensor_prims[0], UsdGeom.Camera) - - # Simulate for a few steps - # note: This is a workaround to ensure that the textures are loaded. - # Check "Known Issues" section in the documentation for more details. - for _ in range(5): - self.sim.step() - - # Simulate physics - for _ in range(10): - # perform rendering - self.sim.step() - # update camera - lidar.update(self.dt, force_recompute=True) - # check info data - for info_key, info_value in lidar.data.info[0].items(): - self.assertTrue(info_key in RTX_LIDAR_INFO_FIELDS.keys()) - self.assertTrue(isinstance(info_value, RTX_LIDAR_INFO_FIELDS[info_key])) - - # check lidar data - for data_key, data_value in lidar.data.output.items(): - if data_key in self.lidar_cfg.optional_data_types: - self.assertTrue(data_value.shape[1] > 0) - - def test_lidar_init_offset(self): - """Test lidar offset configuration.""" - lidar_cfg_offset = copy.deepcopy(self.lidar_cfg) - lidar_cfg_offset.offset = RtxLidarCfg.OffsetCfg(pos=POSITION, rot=QUATERNION) - lidar_cfg_offset.prim_path = "/World/LidarOffset" - lidar = RtxLidar(lidar_cfg_offset) - - # Play sim - self.sim.reset() - - # Retrieve lidar pose using USD API - prim_tf = lidar._sensor_prims[0].ComputeLocalToWorldTransform(Usd.TimeCode.Default()) - prim_tf = np.transpose(prim_tf) - - # check that transform is set correctly - np.testing.assert_allclose(prim_tf[0:3, 3], lidar_cfg_offset.offset.pos) - np.testing.assert_allclose( - convert_quat(tf.Rotation.from_matrix(prim_tf[:3, :3]).as_quat(), "wxyz"), - lidar_cfg_offset.offset.rot, - rtol=1e-5, - atol=1e-5, - ) - - # Simulate for a few steps - # note: This is a workaround to ensure that the textures are loaded. - # Check "Known Issues" section in the documentation for more details. - for _ in range(5): - self.sim.step() - - lidar.update(self.dt) - - def test_lidar_init_multiple(self): - """Test multiple lidar initialization and check info and data outputs.""" - - self.sim._app_control_on_stop_handle = None - lidar_cfg_1 = copy.deepcopy(self.lidar_cfg) - lidar_cfg_1.prim_path = "/World/Lidar1" - lidar_1 = RtxLidar(lidar_cfg_1) - - lidar_cfg_2 = copy.deepcopy(self.lidar_cfg) - lidar_cfg_2.prim_path = "/World/Lidar2" - lidar_2 = RtxLidar(lidar_cfg_2) - - # play sim - self.sim.reset() - - # Simulate for a few steps - # note: This is a workaround to ensure that the textures are loaded. - # Check "Known Issues" section in the documentation for more details. - for _ in range(5): - self.sim.step() - # Simulate physics - for i in range(10): - # perform rendering - self.sim.step() - # update lidar - lidar_1.update(self.dt) - lidar_2.update(self.dt) - # check lidar info - for lidar_info_key in lidar_1.data.info[0].keys(): - info1 = lidar_1.data.info[0][lidar_info_key] - info2 = lidar_2.data.info[0][lidar_info_key] - if isinstance(info1, torch.Tensor): - torch.testing.assert_close(info1, info2) + # Simulate for a few steps + # note: This is a workaround to ensure that the textures are loaded. + # Check "Known Issues" section in the documentation for more details. + for _ in range(5): + sim.step() + + lidar.update(sim.get_physics_dt()) + + +@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +def test_multiple_lidar_init(sim, device, lidar_cfg): + """Test multiple lidar initialization and check info and data outputs.""" + + sim._app_control_on_stop_handle = None + lidar_cfg_1 = copy.deepcopy(lidar_cfg) + lidar_cfg_1.prim_path = "/World/Lidar1" + lidar_1 = RtxLidar(lidar_cfg_1) + + lidar_cfg_2 = copy.deepcopy(lidar_cfg) + lidar_cfg_2.prim_path = "/World/Lidar2" + lidar_2 = RtxLidar(lidar_cfg_2) + + # play sim + sim.reset() + + # Simulate for a few steps + # note: This is a workaround to ensure that the textures are loaded. + # Check "Known Issues" section in the documentation for more details. + for _ in range(5): + sim.step() + # Simulate physics + for i in range(10): + # perform rendering + sim.step() + # update lidar + lidar_1.update(sim.get_physics_dt()) + lidar_2.update(sim.get_physics_dt()) + # check lidar info + for lidar_info_key in lidar_1.data.info[0].keys(): + info1 = lidar_1.data.info[0][lidar_info_key] + info2 = lidar_2.data.info[0][lidar_info_key] + if isinstance(info1, torch.Tensor): + torch.testing.assert_close(info1, info2) + else: + if lidar_info_key == "renderProductPath": + assert info1 == info2.split("_")[0] else: - if lidar_info_key == "renderProductPath": - self.assertTrue(info1 == info2.split("_")[0]) - else: - self.assertTrue(info1 == info2) - # check lidar data shape - for lidar_data_key in lidar_1.data.output.keys(): - data1 = lidar_1.data.output[lidar_data_key] - data2 = lidar_2.data.output[lidar_data_key] - # print(i,"Frame1: ",lidar_1.frame) - # print(i,"Frame2: ",lidar_2.frame) - # print(i,f"Key: {lidar_data_key}, Shape 1: {data1.shape}, Shape 2: {data2.shape}") - self.assertTrue( - data1.shape == data2.shape, f"Key: {lidar_data_key}, Shape 1: {data1.shape}, Shape 2: {data2.shape}" - ) - - def test_lidar_init_custom(self): - """Test custom lidar initialization, data population, and cleanup.""" - # Create custom lidar profile dictionary - with open(EXAMPLE_ROTARY_PATH) as json_file: - sensor_profile = json.load(json_file) - - custom_lidar_cfg = copy.deepcopy(self.lidar_cfg) - custom_lidar_cfg.spawn = sim_utils.LidarCfg(lidar_type="Custom", sensor_profile=sensor_profile) - # Create custom lidar - lidar = RtxLidar(cfg=custom_lidar_cfg) - # Check simulation parameter is set correctly - self.assertTrue(self.sim.has_rtx_sensors()) - # Play sim - self.sim.reset() - # Check if lidar is initialized - self.assertTrue(lidar.is_initialized) - # Check if lidar prim is set correctly and that it is a camera prim - self.assertEqual(lidar._sensor_prims[0].GetPath().pathString, self.lidar_cfg.prim_path) - self.assertIsInstance(lidar._sensor_prims[0], UsdGeom.Camera) - - # Simulate for a few steps - # note: This is a workaround to ensure that the textures are loaded. - # Check "Known Issues" section in the documentation for more details. - for _ in range(5): - self.sim.step() - - # Simulate physics - for _ in range(10): - # perform rendering - self.sim.step() - # update camera - lidar.update(self.dt, force_recompute=True) - # check info data - for info_key, info_value in lidar.data.info[0].items(): - self.assertTrue(info_key in RTX_LIDAR_INFO_FIELDS.keys()) - self.assertTrue(isinstance(info_value, RTX_LIDAR_INFO_FIELDS[info_key])) - - # check lidar data - for data_key, data_value in lidar.data.output.items(): - if data_key in self.lidar_cfg.optional_data_types: - self.assertTrue(data_value.shape[1] > 0) - - del lidar - - # check proper file cleanup - custom_profile_name = self.lidar_cfg.spawn.sensor_profile_temp_prefix - custom_profile_dir = self.lidar_cfg.spawn.sensor_profile_temp_dir - files = os.listdir(custom_profile_dir) - for file in files: - self.assertTrue( - custom_profile_name not in file, msg=f"{custom_profile_name} found in {custom_profile_dir}/{file}" - ) + assert info1 == info2 + # check lidar data shape both instances should produce the same amount of data + for lidar_data_key in lidar_1.data.output.keys(): + data1 = lidar_1.data.output[lidar_data_key] + data2 = lidar_2.data.output[lidar_data_key] + assert data1.shape == data2.shape + + +@pytest.mark.parametrize("device", ["cuda:0", "cpu"]) +def test_custom_lidar_init(sim, device, lidar_cfg): + """Test custom lidar initialization, data population, and cleanup.""" + # Create custom lidar profile dictionary + with open(EXAMPLE_ROTARY_PATH) as json_file: + sensor_profile = json.load(json_file) + + custom_lidar_cfg = copy.deepcopy(lidar_cfg) + custom_lidar_cfg.spawn = sim_utils.LidarCfg(lidar_type="Custom", sensor_profile=sensor_profile) + # Create custom lidar + lidar = RtxLidar(cfg=custom_lidar_cfg) + # Check simulation parameter is set correctly + assert sim.has_rtx_sensors() + # Play sim + sim.reset() + # Check if lidar is initialized + assert lidar.is_initialized + # Check if lidar prim is set correctly and that it is a camera prim + assert lidar._sensor_prims[0].GetPath().pathString == lidar_cfg.prim_path + assert isinstance(lidar._sensor_prims[0], UsdGeom.Camera) + + # Simulate for a few steps + # note: This is a workaround to ensure that the textures are loaded. + # Check "Known Issues" section in the documentation for more details. + for _ in range(5): + sim.step() + + # Simulate physics + for _ in range(10): + # perform rendering + sim.step() + # update camera + lidar.update(sim.get_physics_dt(), force_recompute=True) + # check info data + for info_key, info_value in lidar.data.info[0].items(): + assert info_key in RTX_LIDAR_INFO_FIELDS.keys() + assert isinstance(info_value, RTX_LIDAR_INFO_FIELDS[info_key]) + + # check lidar data + for data_key, data_value in lidar.data.output.items(): + if data_key in lidar_cfg.optional_data_types: + assert data_value.shape[1] > 0 + + del lidar + + # check proper file cleanup + custom_profile_name = lidar_cfg.spawn.sensor_profile_temp_prefix + custom_profile_dir = lidar_cfg.spawn.sensor_profile_temp_dir + files = os.listdir(custom_profile_dir) + for file in files: + assert custom_profile_name not in file if __name__ == "__main__": - run_tests() + pytest.main([__file__, "-v", "--maxfail=1"]) From 511fda38cceb6033b36b9aeef0e0ff60bbd7265b Mon Sep 17 00:00:00 2001 From: Samir Chowdhury Date: Wed, 28 May 2025 14:23:35 -0700 Subject: [PATCH 06/12] Adding RTX LiDAR to mdp observations --- .../isaaclab/envs/mdp/observations.py | 20 ++++++++++++++++++- source/isaaclab/isaaclab/sensors/__init__.py | 1 + 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/source/isaaclab/isaaclab/envs/mdp/observations.py b/source/isaaclab/isaaclab/envs/mdp/observations.py index 56ce0228f2f..df55d187a4d 100644 --- a/source/isaaclab/isaaclab/envs/mdp/observations.py +++ b/source/isaaclab/isaaclab/envs/mdp/observations.py @@ -19,7 +19,7 @@ from isaaclab.managers import SceneEntityCfg from isaaclab.managers.manager_base import ManagerTermBase from isaaclab.managers.manager_term_cfg import ObservationTermCfg -from isaaclab.sensors import Camera, Imu, RayCaster, RayCasterCamera, TiledCamera +from isaaclab.sensors import Camera, Imu, RayCaster, RayCasterCamera, RtxLidar, TiledCamera if TYPE_CHECKING: from isaaclab.envs import ManagerBasedEnv, ManagerBasedRLEnv @@ -300,6 +300,24 @@ def imu_lin_acc(env: ManagerBasedEnv, asset_cfg: SceneEntityCfg = SceneEntityCfg return asset.data.lin_acc_b +def point_cloud( + env: ManagerBasedEnv, sensor_cfg: SceneEntityCfg = SceneEntityCfg("lidar"), data_type: str = "data" +) -> torch.Tensor: + """Lidar point cloud data + + Args: + env: The environment. + sensor_cfg: The desired sensor to read from. Defaults to SceneEntityCfg("lidar"). + data_type: The data type to pull from the desired lidar. Defaults to "data". + + Returns: + The point cloud in the sensor frame. Shape is (num_envs, idk) + """ + + sensor: RtxLidar = env.scene[sensor_cfg.name] + return sensor.data.output[data_type] + + def image( env: ManagerBasedEnv, sensor_cfg: SceneEntityCfg = SceneEntityCfg("tiled_camera"), diff --git a/source/isaaclab/isaaclab/sensors/__init__.py b/source/isaaclab/isaaclab/sensors/__init__.py index 0f28b6f0573..37b2f03a18f 100644 --- a/source/isaaclab/isaaclab/sensors/__init__.py +++ b/source/isaaclab/isaaclab/sensors/__init__.py @@ -40,5 +40,6 @@ from .frame_transformer import * # noqa: F401 from .imu import * # noqa: F401, F403 from .ray_caster import * # noqa: F401, F403 +from .rtx_lidar import * # noqa: F401, F403 from .sensor_base import SensorBase # noqa: F401 from .sensor_base_cfg import SensorBaseCfg # noqa: F401 From 845b5d52c51ac536e0ff024b3577156b22e8f288 Mon Sep 17 00:00:00 2001 From: Samir Chowdhury Date: Wed, 28 May 2025 14:51:09 -0700 Subject: [PATCH 07/12] Fixing circular import --- .../isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py | 2 +- .../isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py index 236b3c00047..5d1d514a4c0 100644 --- a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py @@ -8,10 +8,10 @@ from dataclasses import MISSING from typing import Literal -from isaaclab.sensors import SensorBaseCfg from isaaclab.sim import LidarCfg from isaaclab.utils import configclass +from ..sensor_base_cfg import SensorBaseCfg from .rtx_lidar import RtxLidar diff --git a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py index 67f6961d29f..7dd51969676 100644 --- a/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py +++ b/source/isaaclab/isaaclab/sim/spawners/sensors/sensors_cfg.py @@ -9,8 +9,8 @@ from collections.abc import Callable from typing import Any, Literal -from isaaclab import ISAACLAB_EXT_DIR import isaaclab.utils.sensors as sensor_utils +from isaaclab import ISAACLAB_EXT_DIR from isaaclab.sim.spawners.spawner_cfg import SpawnerCfg from isaaclab.utils import configclass @@ -144,8 +144,7 @@ def from_intrinsic_matrix( """ # raise not implemented error is projection type is not pinhole if projection_type != "pinhole": - raise NotImplementedError( - "Only pinhole projection type is supported.") + raise NotImplementedError("Only pinhole projection type is supported.") usd_camera_params = sensor_utils.convert_camera_intrinsics_to_usd( intrinsic_matrix=intrinsic_matrix, height=height, width=width, focal_length=focal_length @@ -304,8 +303,7 @@ class LidarType: """Custom lidar parameters to use if lidar_type="Custom" see https://docs.omniverse.nvidia.com/kit/docs/omni.sensors.nv.lidar/latest/lidar_extension.html""" - sensor_profile_temp_dir: str = os.path.abspath( - os.path.join(ISAACLAB_EXT_DIR, "isaaclab/sensors/rtx_lidar")) + sensor_profile_temp_dir: str = os.path.abspath(os.path.join(ISAACLAB_EXT_DIR, "isaaclab/sensors/rtx_lidar")) """The location of the generated custom sensor profile json file.""" sensor_profile_temp_prefix: str = "Temp_Config_" From 209133644f26beaed1d3ce47c1b6ba669dbc9e02 Mon Sep 17 00:00:00 2001 From: Samir Chowdhury Date: Wed, 18 Jun 2025 13:31:24 -0700 Subject: [PATCH 08/12] Addressing comments in https://github.com/isaac-sim/IsaacLab/pull/2308 --- .../isaaclab/envs/mdp/observations.py | 2 +- .../isaaclab/sensors/rtx_lidar/rtx_lidar.py | 50 +++++++++---------- .../sensors/rtx_lidar/rtx_lidar_cfg.py | 6 +-- 3 files changed, 27 insertions(+), 31 deletions(-) diff --git a/source/isaaclab/isaaclab/envs/mdp/observations.py b/source/isaaclab/isaaclab/envs/mdp/observations.py index f3331eaa303..aec3af82717 100644 --- a/source/isaaclab/isaaclab/envs/mdp/observations.py +++ b/source/isaaclab/isaaclab/envs/mdp/observations.py @@ -311,7 +311,7 @@ def point_cloud( data_type: The data type to pull from the desired lidar. Defaults to "data". Returns: - The point cloud in the sensor frame. Shape is (num_envs, idk) + The point cloud in the sensor frame. Shape is (num_envs, num_points, 3) """ sensor: RtxLidar = env.scene[sensor_cfg.name] diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py index 76d3c628a3a..8605ba81617 100644 --- a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py @@ -83,7 +83,7 @@ def __init__(self, cfg: RtxLidarCfg): self._data = RtxLidarData() def __del__(self): - """Unsubscribes from callbacks and detach from the replicator registry and clean up any custom lidar configs.""" + """Unsubscribes from callbacks and detach from the replicator registry and clean up any custom rtx lidar configs.""" # unsubscribe callbacks super().__del__() # delete from replicator registry @@ -122,7 +122,7 @@ def frame(self) -> torch.tensor: @property def render_product_paths(self) -> list[str]: - """The path of the render products for the cameras. + """The path of the render products for the RTX LiDAR. This can be used via replicator interfaces to attach to writes or external annotator registry. """ @@ -135,7 +135,7 @@ def render_product_paths(self) -> list[str]: def reset(self, env_ids: Sequence[int] | None = None): if not self._is_initialized: raise RuntimeError( - "Camera could not be initialized. Please ensure --enable_cameras is used to enable rendering." + "RTX LiDAR could not be initialized. Please ensure --enable_cameras is used to enable rendering." ) # reset the timestamps super().reset(env_ids) @@ -164,7 +164,7 @@ def _initialize_impl(self): carb_settings_iface = carb.settings.get_settings() if not carb_settings_iface.get("/isaaclab/cameras_enabled"): raise RuntimeError( - "A camera was spawned without the --enable_cameras flag. Please use --enable_cameras to enable" + "A RTX LiDAR was spawned without the --enable_cameras flag. Please use --enable_cameras to enable" " rendering." ) @@ -199,7 +199,7 @@ def _initialize_impl(self): lidar_prim = stage.GetPrimAtPath(lidar_prim_path) # Check if prim is a camera if not lidar_prim.IsA(UsdGeom.Camera): - raise RuntimeError(f"Prim at path '{lidar_prim_path}' is not a Camera.") + raise RuntimeError(f"Prim at path '{lidar_prim_path}' is not a Camera (which is the base prim for RTXLiDAR).") # Add to list sensor_prim = UsdGeom.Camera(lidar_prim) self._sensor_prims.append(sensor_prim) @@ -216,27 +216,27 @@ def _initialize_impl(self): "outputTimestamp": True, # always turn on timestamp field } + DATA_TYPE_TO_PARAM_KEY = { + "azimuth": "outputAzimuth", + "elevation": "outputElevation", + "normal": "outputNormal", + "velocity": "outputVelocity", + "beamId": "outputBeamId", + "emitterId": "outputEmitterId", + "materialId": "outputMaterialId", + "objectId": "outputObjectId", + } + # create annotator node annotator_type = "RtxSensorCpuIsaacCreateRTXLidarScanBuffer" rep_annotator = rep.AnnotatorRegistry.get_annotator(annotator_type) + # turn on any optional data type returns + for name in self.cfg.optional_data_types: - if name == "azimuth": - init_params["outputAzimuth"] = True - elif name == "elevation": - init_params["outputElevation"] = True - elif name == "normal": - init_params["outputNormal"] = True - elif name == "velocity": - init_params["outputVelocity"] = True - elif name == "beamId": - init_params["outputBeamId"] = True - elif name == "emitterId": - init_params["outputEmitterId"] = True - elif name == "materialId": - init_params["outputMaterialId"] = True - elif name == "objectId": - init_params["outputObjectId"] = True + param_key = DATA_TYPE_TO_PARAM_KEY.get(name) + if param_key: + init_params[param_key] = True # transform the data output to be relative to sensor frame if self.cfg.data_frame == "sensor": @@ -312,7 +312,7 @@ def _update_buffers_impl(self, env_ids: Sequence[int]): def _process_annotator_output(self, name: str, output: Any) -> tuple[torch.tensor, dict | None]: """Process the annotator output. - This function is called after the data has been collected from all the cameras. + This function is called after the data has been collected from all the RTXLiDARs. """ # extract info and data from the output if isinstance(output, dict): @@ -321,13 +321,9 @@ def _process_annotator_output(self, name: str, output: Any) -> tuple[torch.tenso else: data = output info = None + # convert data into torch tensor data = convert_to_torch(data, device=self.device) - - # process data for different segmentation types - # Note: Replicator returns raw buffers of dtype int32 for segmentation types - # so we need to convert them to uint8 4 channel images for colorized types - return data, info """ diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py index 5d1d514a4c0..44a0f64a3c5 100644 --- a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py @@ -45,11 +45,11 @@ class OffsetCfg: Please refer to the :class:'RtxLidar' and :class:'RtxLidarData' for a list and description of available data types. """ data_frame: Literal["world", "sensor"] = "world" - """The frame to represent the output.data. + """The frame to represent the :attr:`RtxLidar.data` in. - If 'world' the output.data will be in the world frame. If 'sensor' the output.data will be in the sensor frame. + If 'world' the :attr:`RtxLidar.data` will be in the world frame. If 'sensor' the :attr:`RtxLidar.data` will be in the sensor frame. """ - spawn: LidarCfg = MISSING + spawn: LidarCfg | None = None """Spawn configuration for the asset. If None, then the prim is not spawned by the asset. Instead, it is assumed that the From ad39dcad3ade8c90b1dea3b69baea26dba7dfc70 Mon Sep 17 00:00:00 2001 From: Samir Chowdhury Date: Wed, 18 Jun 2025 14:17:46 -0700 Subject: [PATCH 09/12] Running formatter --- CONTRIBUTORS.md | 1 + source/isaaclab/isaaclab/sensors/rtx_lidar/__init__.py | 2 +- source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py | 6 ++++-- source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py | 3 +-- .../isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_data.py | 2 +- source/isaaclab/test/sensors/test_rtx_lidar.py | 2 +- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 90b9befa261..9a13c517a6f 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -101,6 +101,7 @@ Guidelines for modifications: * Ritvik Singh * Rosario Scalise * Ryley McCarroll +* Samir Chowdhury * Shafeef Omar * Shundo Kishi * Stefan Van de Mosselaer diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/__init__.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/__init__.py index 50697595980..74bd9cb25a7 100644 --- a/source/isaaclab/isaaclab/sensors/rtx_lidar/__init__.py +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). # All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py index 8605ba81617..1f01d13516a 100644 --- a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). # All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause @@ -199,7 +199,9 @@ def _initialize_impl(self): lidar_prim = stage.GetPrimAtPath(lidar_prim_path) # Check if prim is a camera if not lidar_prim.IsA(UsdGeom.Camera): - raise RuntimeError(f"Prim at path '{lidar_prim_path}' is not a Camera (which is the base prim for RTXLiDAR).") + raise RuntimeError( + f"Prim at path '{lidar_prim_path}' is not a Camera (which is the base prim for RTXLiDAR)." + ) # Add to list sensor_prim = UsdGeom.Camera(lidar_prim) self._sensor_prims.append(sensor_prim) diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py index 44a0f64a3c5..36695a00eec 100644 --- a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_cfg.py @@ -1,11 +1,10 @@ -# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). # All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause from __future__ import annotations -from dataclasses import MISSING from typing import Literal from isaaclab.sim import LidarCfg diff --git a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_data.py b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_data.py index 1102d5188b3..f9d68beb38c 100644 --- a/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_data.py +++ b/source/isaaclab/isaaclab/sensors/rtx_lidar/rtx_lidar_data.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). # All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause diff --git a/source/isaaclab/test/sensors/test_rtx_lidar.py b/source/isaaclab/test/sensors/test_rtx_lidar.py index 51e84d4b020..e4327adb55e 100644 --- a/source/isaaclab/test/sensors/test_rtx_lidar.py +++ b/source/isaaclab/test/sensors/test_rtx_lidar.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2025, The Isaac Lab Project Developers. +# Copyright (c) 2022-2025, The Isaac Lab Project Developers (https://github.com/isaac-sim/IsaacLab/blob/main/CONTRIBUTORS.md). # All rights reserved. # # SPDX-License-Identifier: BSD-3-Clause From 01ea0ea028af7f82f05e80f157a805492623f0e3 Mon Sep 17 00:00:00 2001 From: Samir Chowdhury Date: Wed, 18 Jun 2025 14:17:46 -0700 Subject: [PATCH 10/12] Running formatter --- source/isaaclab/docs/CHANGELOG.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/source/isaaclab/docs/CHANGELOG.rst b/source/isaaclab/docs/CHANGELOG.rst index aa1cb1bc068..81a812a201c 100644 --- a/source/isaaclab/docs/CHANGELOG.rst +++ b/source/isaaclab/docs/CHANGELOG.rst @@ -4,6 +4,12 @@ Changelog 0.40.6 (2025-06-12) ~~~~~~~~~~~~~~~~~~~ +Added +^^^^^ + +* Added RTX LiDAR `~isaaclab.sensor` and corresponding observations in `~isaaclab.mdp.observations` + + Fixed ^^^^^ From ccec813b2b539fad7dd74096952883fe3a097e43 Mon Sep 17 00:00:00 2001 From: Samir Chowdhury Date: Wed, 18 Jun 2025 14:17:46 -0700 Subject: [PATCH 11/12] Running formatter --- source/isaaclab/config/extension.toml | 2 +- source/isaaclab/docs/CHANGELOG.rst | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/source/isaaclab/config/extension.toml b/source/isaaclab/config/extension.toml index aa5e112813c..28c2225b213 100644 --- a/source/isaaclab/config/extension.toml +++ b/source/isaaclab/config/extension.toml @@ -1,7 +1,7 @@ [package] # Note: Semantic Versioning is used: https://semver.org/ -version = "0.40.6" +version = "0.40.7" # Description title = "Isaac Lab framework for Robot Learning" diff --git a/source/isaaclab/docs/CHANGELOG.rst b/source/isaaclab/docs/CHANGELOG.rst index 81a812a201c..9306a787b99 100644 --- a/source/isaaclab/docs/CHANGELOG.rst +++ b/source/isaaclab/docs/CHANGELOG.rst @@ -1,13 +1,22 @@ Changelog --------- +0.40.7 (2025-06-18) +~~~~~~~~~~~~~~~~~~~ + +Added +^^^^^ + +* Added RTX LiDAR :class:`~isaaclab.sensors.rtx_lidar.rtx_lidar` and corresponding observations in :class:`~isaaclab.envs.mdp.observations` + + 0.40.6 (2025-06-12) ~~~~~~~~~~~~~~~~~~~ Added ^^^^^ -* Added RTX LiDAR `~isaaclab.sensor` and corresponding observations in `~isaaclab.mdp.observations` +* Added RTX LiDAR :class:`~isaaclab.sensors.rtx_lidar.rtx_lidar` and corresponding observations in :class:`~isaaclab.envs.mdp.observations` Fixed From 873c88ad891a03dbab25bf009300799d1514214a Mon Sep 17 00:00:00 2001 From: Samir Chowdhury Date: Wed, 18 Jun 2025 14:42:48 -0700 Subject: [PATCH 12/12] Removing extra changelog line --- source/isaaclab/docs/CHANGELOG.rst | 6 ------ 1 file changed, 6 deletions(-) diff --git a/source/isaaclab/docs/CHANGELOG.rst b/source/isaaclab/docs/CHANGELOG.rst index 9306a787b99..d03373a4810 100644 --- a/source/isaaclab/docs/CHANGELOG.rst +++ b/source/isaaclab/docs/CHANGELOG.rst @@ -13,12 +13,6 @@ Added 0.40.6 (2025-06-12) ~~~~~~~~~~~~~~~~~~~ -Added -^^^^^ - -* Added RTX LiDAR :class:`~isaaclab.sensors.rtx_lidar.rtx_lidar` and corresponding observations in :class:`~isaaclab.envs.mdp.observations` - - Fixed ^^^^^