Skip to content

Fix illegal memory access for textureless meshes #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

Fix illegal memory access for textureless meshes #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions ..._ros_gxf_extensions/gxf_isaac_foundationpose/gxf/foundationpose/foundationpose_render.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -607,11 +607,12 @@ gxf_result_t FoundationposeRender::NvdiffrastRender(
CHECK_CUDA_ERRORS(cudaGetLastError());
} else {
auto float_texture_map_data = float_texture_map_tensor_.exportData<nvcv::TensorDataStridedCuda>();
// The texture map is a list of vertex colors, needs broadcasting in interpolate to avoid illegal memory access
interpolate(
cuda_stream,
reinterpret_cast<float*>(float_texture_map_data->basePtr()), rast_out_device_, mesh_data_ptr->mesh_faces_device, color_device_,
mesh_data_ptr->num_vertices, mesh_data_ptr->num_faces, kVertexPoints,
H, W, N);
H, W, N, 1);
CHECK_CUDA_ERRORS(cudaGetLastError());
}

Expand Down Expand Up @@ -833,7 +834,11 @@ gxf_result_t FoundationposeRender::tick() noexcept {
// Get mesh data from sampling component if not already set
auto mesh_data_ptr = mesh_storage_.get()->GetMeshData();

if (float_texture_map_tensor_.empty() || texture_path_cache_ != mesh_data_ptr->texture_path) {
// For pure color texture (vetex colors), we can reuse the cached version if the size matches
if (float_texture_map_tensor_.empty() || texture_path_cache_ != mesh_data_ptr->texture_path
|| mesh_data_ptr->texture_map_height != float_texture_map_tensor_.shape()[1]
|| mesh_data_ptr->texture_map_width != float_texture_map_tensor_.shape()[2]
|| mesh_data_ptr->texture_map_channels != float_texture_map_tensor_.shape()[3]) {
NormalizeImage(
cuda_stream_,
mesh_data_ptr->texture_map_device,
Expand Down
4 changes: 2 additions & 2 deletions ...c_ros_gxf_extensions/gxf_isaac_foundationpose/gxf/foundationpose/foundationpose_render.cu
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@ void rasterize(

void interpolate(
cudaStream_t stream, float* attr_ptr, float* rast_ptr, int32_t* tri_ptr, float* out, int num_vertices,
int num_triangles, int attr_dim, int H, int W, int C) {
int num_triangles, int attr_dim, int H, int W, int C, int attr_bc) {
int instance_mode = attr_dim > 2 ? 1 : 0;

InterpolateKernelParams p = {}; // Initialize all fields to zero.
Expand All @@ -346,7 +346,7 @@ void interpolate(
p.attr = attr_ptr;
p.rast = rast_ptr;
p.tri = tri_ptr;
p.attrBC = 0;
p.attrBC = attr_bc;
p.out = out;

// Choose launch parameters.
Expand Down
2 changes: 1 addition & 1 deletion ...s_gxf_extensions/gxf_isaac_foundationpose/gxf/foundationpose/foundationpose_render.cu.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ void rasterize(
int H, int W, int C);
void interpolate(
cudaStream_t stream, float* attr_ptr, float* rast_ptr, int32_t* tri_ptr, float* out, int num_vertices,
int num_triangles, int attr_dim, int H, int W, int C);
int num_triangles, int attr_dim, int H, int W, int C, int attr_bc = 0);
void texture(
cudaStream_t stream, float* tex_ptr, float* uv_ptr, float* out, int tex_height, int tex_width, int tex_channel,
int tex_depth, int H, int W, int N);
Expand Down
14 changes: 7 additions & 7 deletions isaac_ros_gxf_extensions/gxf_isaac_foundationpose/gxf/foundationpose/mesh_storage.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@
#include "foundationpose_utils.hpp"

namespace {
constexpr int kFixTextureMapWidth = 1920;
constexpr int kFixTextureMapHeight = 1080;
constexpr int kFixTextureMapColor = 128;
} // namespace

Expand Down Expand Up @@ -87,12 +85,13 @@ gxf_result_t MeshStorage::LoadTextureData(const std::string& texture_file_path)
cv::Mat rgb_texture_map;

if (!std::filesystem::exists(texture_file_path)) {
if (mesh_data_->texture_path.empty() && mesh_data_->texture_map_device != nullptr) {
if (mesh_data_->texture_path.empty() && mesh_data_->texture_map_device != nullptr && mesh_data_->texture_map_width == mesh_data_->num_vertices) {
GXF_LOG_WARNING("[MeshStorage] %s could not be found, reuse the pure color texture map", texture_file_path.c_str());
return GXF_SUCCESS;
}
GXF_LOG_WARNING("[MeshStorage], %s could not be found, assign texture map with pure color", texture_file_path.c_str());
rgb_texture_map = cv::Mat(kFixTextureMapHeight, kFixTextureMapWidth, CV_8UC3,
GXF_LOG_WARNING("[MeshStorage] %s could not be found, assign texture map with pure color", texture_file_path.c_str());
// The pure color texture map is actually a list of vertex colors
rgb_texture_map = cv::Mat(1, mesh_data_->num_vertices, CV_8UC3,
cv::Scalar(kFixTextureMapColor, kFixTextureMapColor, kFixTextureMapColor));
mesh_data_->has_tex = false;
} else {
Expand Down Expand Up @@ -296,8 +295,9 @@ gxf_result_t MeshStorage::LoadMeshData(const std::string& mesh_file_path) {
GXF_LOG_WARNING("[MeshStorage] No material index found for mesh");
}

// Only reload texture if the path has changed
if (texture_path_str != mesh_data_->texture_path || mesh_data_->texture_map_device == nullptr) {
// Only reload texture if the path has changed or if no texture was loaded before
if (texture_path_str != mesh_data_->texture_path || mesh_data_->texture_map_device == nullptr
|| !mesh_data_->has_tex) {
GXF_LOG_DEBUG("[MeshStorage] Texture path has changed, reloading texture");
auto result = LoadTextureData(texture_path_str);
if (result != GXF_SUCCESS) {
Expand Down