Add an example client

Author: AndyZe

CMakeLists.txt

@@ -17,17 +17,24 @@ endforeach()
 
 find_package(ament_cmake REQUIRED)
 
+include_directories(include)
+
 add_executable(openai_server src/openai_server.cpp)
 ament_target_dependencies(openai_server ${THIS_PACKAGE_INCLUDE_DEPENDS})
 target_link_libraries(openai_server b64 CURL::libcurl nlohmann_json
                       ${OpenCV_LIBS})
 
+add_executable(example_client src/example_client.cpp)
+ament_target_dependencies(example_client ${THIS_PACKAGE_INCLUDE_DEPENDS})
+target_link_libraries(example_client ${OpenCV_LIBS})
+
 install(
-  TARGETS openai_server
+  TARGETS example_client openai_server
   ARCHIVE DESTINATION lib
   LIBRARY DESTINATION lib
   RUNTIME DESTINATION lib/ros2_openai_server)
 
+install(DIRECTORY include DESTINATION include/ros2_openai_server)
 install(
   DIRECTORY test_data
   DESTINATION DESTINATION

README.md

@@ -22,6 +22,10 @@ Here's an example that returns a full string.
 
 `ros2 service call /openai_string_response ai_msgs/srv/StringResponse prompt:\ "are you a pirate?"`
 
+There's an example client which sends an image of a wooden table and prompts whether it is indeed a wooden table:
+
+`ros2 run ros2_openai_server example_client`
+
 ## Citation
 
 If you use this work, please cite it like so:

include/ros2_openai_server/convert_image_to_base_64.hpp

@@ -0,0 +1,62 @@
+#pragma once
+
+#include <b64/encode.h>
+#include <cv_bridge/cv_bridge.h>
+#include <iostream>
+#include <opencv2/opencv.hpp>
+#include <optional>
+
+namespace openai_server
+{
+/**
+ * @brief Convert jpeg to base64 encoding. There are 2 possible input types.
+ *
+ * @param image_path path to image on disk
+ * @param image_cv_mat an image
+ * @return image encoded as string. If there was an error, it will be empty.
+ */
+std::string convert_image_to_base_64(const std::optional<const std::string>& image_path,
+                                     const std::optional<const cv::Mat>& image_cv_mat)
+{
+  if (!image_path && !image_cv_mat)
+  {
+    std::cerr << "Both inputs to convert_image_to_base_64() are nullopt. One valid input is required." << std::endl;
+    return "";
+  }
+
+  cv::Mat image;
+  if (image_path)
+  {
+    image = cv::imread(*image_path, cv::IMREAD_COLOR);
+    if (image.empty())
+    {
+      std::cerr << "Could not read the image: " << *image_path << std::endl;
+      return "";
+    }
+  }
+  else
+  {
+    image = image_cv_mat.value();
+    if (image.empty())
+    {
+      std::cerr << "Passed image was empty" << std::endl;
+      return "";
+    }
+  }
+
+  // Encode image to JPEG format
+  std::vector<uchar> buf;
+  cv::imencode(".jpg", image, buf);
+
+  // Convert the buffer to a char array
+  std::string encoded_data(buf.begin(), buf.end());
+
+  // Encode to Base64 using libb64
+  base64::encoder E;
+  std::ostringstream os;
+  std::istringstream is(encoded_data);
+  E.encode(is, os);
+
+  return os.str();
+}
+}  // namespace openai_server

src/example_client.cpp

@@ -0,0 +1,52 @@
+#include "ai_msgs/srv/bool_response.hpp"
+#include "rclcpp/rclcpp.hpp"
+
+#include <ament_index_cpp/get_package_share_directory.hpp>
+#include <chrono>
+#include <cstdlib>
+#include <cv_bridge/cv_bridge.h>
+#include <memory>
+#include <opencv2/opencv.hpp>
+
+using namespace std::chrono_literals;
+
+int main(int argc, char** argv)
+{
+  rclcpp::init(argc, argv);
+
+  std::shared_ptr<rclcpp::Node> node = rclcpp::Node::make_shared("open_ai_client");
+  rclcpp::Client<ai_msgs::srv::BoolResponse>::SharedPtr client =
+      node->create_client<ai_msgs::srv::BoolResponse>("openai_bool_response");
+
+  auto request = std::make_shared<ai_msgs::srv::BoolResponse::Request>();
+  request->prompt = "Is this a wooden table? Please respond in one word, yes or no.";
+
+  // Add an image of a wooden table to the OpenAI request
+  std::string pkg_share_directory = ament_index_cpp::get_package_share_directory("ros2_openai_server");
+  std::string image_path = pkg_share_directory + "/test_data/wood_table.jpg";
+  cv::Mat img = cv::imread(image_path, cv::IMREAD_COLOR);
+  sensor_msgs::msg::Image::SharedPtr msg = cv_bridge::CvImage(std_msgs::msg::Header(), "bgr8", img).toImageMsg();
+  request->image = *msg;
+
+  while (!client->wait_for_service(5s))
+  {
+    if (!rclcpp::ok())
+    {
+      RCLCPP_ERROR(rclcpp::get_logger("rclcpp"), "Interrupted while waiting for the service. Exiting.");
+      return 0;
+    }
+  }
+
+  auto result = client->async_send_request(request);
+  // Wait for the result.
+  if (rclcpp::spin_until_future_complete(node, result) == rclcpp::FutureReturnCode::SUCCESS)
+  {
+    RCLCPP_INFO(rclcpp::get_logger("rclcpp"), "Response: %b", result.get()->response);
+  }
+  else
+  {
+    RCLCPP_ERROR(rclcpp::get_logger("rclcpp"), "Failed to call service");
+  }
+
+  return 0;
+}

src/openai_server.cpp

@@ -1,77 +1,18 @@
+#include "ros2_openai_server/convert_image_to_base_64.hpp"
+
 #include "ai_msgs/srv/bool_response.hpp"
 #include "ai_msgs/srv/string_response.hpp"
 #include "rclcpp/rclcpp.hpp"
 #include "sensor_msgs/msg/image.hpp"
 
-#include <b64/encode.h>
+#include <ament_index_cpp/get_package_share_directory.hpp>
 #include <curl/curl.h>
-#include <cv_bridge/cv_bridge.h>
 #include <nlohmann/json.hpp>
-#include <opencv2/opencv.hpp>
 #include <sensor_msgs/image_encodings.hpp>
 #include <stdlib.h>
 
-// Uncomment this to use the test image
-#define HARD_CODED_IMAGE_TESTING
-
 namespace openai_server
 {
-
-namespace
-{
-/**
- * @brief Convert jpeg to base64 encoding. There are 2 possible input types.
- *
- * @param image_path path to image on disk
- * @param image_cv_mat an image
- * @return image encoded as string. If there was an error, it will be empty.
- */
-std::string convert_image_to_base_64(const std::optional<const std::string>& image_path,
-                                     const std::optional<const cv::Mat>& image_cv_mat)
-{
-  if (!image_path && !image_cv_mat)
-  {
-    std::cerr << "Both inputs to convert_image_to_base_64() are nullopt. One valid input is required." << std::endl;
-    return "";
-  }
-
-  cv::Mat image;
-  if (image_path)
-  {
-    image = cv::imread(*image_path, cv::IMREAD_COLOR);
-    if (image.empty())
-    {
-      std::cerr << "Could not read the image: " << *image_path << std::endl;
-      return "";
-    }
-  }
-  else
-  {
-    image = image_cv_mat.value();
-    if (image.empty())
-    {
-      std::cerr << "Passed image was empty" << std::endl;
-      return "";
-    }
-  }
-
-  // Encode image to JPEG format
-  std::vector<uchar> buf;
-  cv::imencode(".jpg", image, buf);
-
-  // Convert the buffer to a char array
-  std::string encoded_data(buf.begin(), buf.end());
-
-  // Encode to Base64 using libb64
-  base64::encoder E;
-  std::ostringstream os;
-  std::istringstream is(encoded_data);
-  E.encode(is, os);
-
-  return os.str();
-}
-}  // namespace
-
 class OpenAIServer : public rclcpp::Node
 {
 public:
@@ -141,38 +82,23 @@ class OpenAIServer : public rclcpp::Node
       request_data["messages"][0]["role"] = "user";
       // Attach the image, if any
       // See https://platform.openai.com/docs/guides/vision
-      if (
-#ifdef HARD_CODED_IMAGE_TESTING
-          1
-#endif
-#ifndef HARD_CODED_IMAGE_TESTING
-          0
-#endif
-      )
-      {
-        std::string encoded_image = convert_image_to_base_64(
-            std::optional<const std::string>(
-                "/home/andy/ws_nav2/install/ros2_openai_server/share/ros2_openai_server/test_data/wood_table.jpg"),
-            std::nullopt);
-        request_data["messages"][0]["content"][0]["type"] = "text";
-        request_data["messages"][0]["content"][0]["text"] = prompt;
-        request_data["messages"][0]["content"][1]["type"] = "image_url";
-        request_data["messages"][0]["content"][1]["image_url"]["url"] = ("data:image/jpg;base64," + encoded_image);
-      }
-      else if (image.height > 0)
+      if (image.height > 0)
       {
-        cv_bridge::CvImagePtr cv_ptr;
+        std::string encoded_image;
         try
         {
-          cv_ptr = cv_bridge::toCvCopy(image, image.encoding);
-          std::string encoded_image =
-              convert_image_to_base_64(std::nullopt, std::optional<const cv::Mat>(cv_ptr->image));
+          cv_bridge::CvImagePtr cv_ptr = cv_bridge::toCvCopy(image, image.encoding);
+          encoded_image = convert_image_to_base_64(std::nullopt, std::optional<const cv::Mat>(cv_ptr->image));
         }
         catch (cv_bridge::Exception& e)
         {
           RCLCPP_ERROR(this->get_logger(), "cv_bridge exception: %s", e.what());
           return false;
         }
+        request_data["messages"][0]["content"][0]["type"] = "text";
+        request_data["messages"][0]["content"][0]["text"] = prompt;
+        request_data["messages"][0]["content"][1]["type"] = "image_url";
+        request_data["messages"][0]["content"][1]["image_url"]["url"] = ("data:image/jpg;base64," + encoded_image);
       }
       else  // No image
       {