feat: Pulsar external stream (#891)

Author: zliang-min

.github/workflows/manual_trigger_build.yml

@@ -225,7 +225,7 @@ jobs:
           export CC=$(brew --prefix llvm@17)/bin/clang
           export CXX=$(brew --prefix llvm@17)/bin/clang++
           export PATH=$(brew --prefix llvm@17)/bin:$PATH
-          cmake -B ${{ env.build_directory }} -G "Ninja" -DCMAKE_BUILD_TYPE=${{ env.build_type }} -DENABLE_TESTS=OFF -DENABLE_UTILS=OFF -DENABLE_EXAMPLES=OFF
+          cmake -B ${{ env.build_directory }} -G "Ninja" -DCMAKE_BUILD_TYPE=${{ env.build_type }} -DENABLE_TESTS=OFF -DENABLE_UTILS=OFF -DENABLE_EXAMPLES=OFF -DENABLE_PULSAR=OFF
 
       - name: Build with Ninja
         run: cmake --build ${{ env.build_directory }}

.github/workflows/release_build.yml

@@ -322,7 +322,7 @@ jobs:
           export CC=$(brew --prefix llvm@17)/bin/clang
           export CXX=$(brew --prefix llvm@17)/bin/clang++
           export PATH=$(brew --prefix llvm@17)/bin:$PATH
-          cmake -B ${{ env.build_directory }} -G "Ninja" -DCMAKE_BUILD_TYPE=${{ env.build_type }} -DENABLE_TESTS=OFF -DENABLE_UTILS=OFF -DENABLE_EXAMPLES=OFF
+          cmake -B ${{ env.build_directory }} -G "Ninja" -DCMAKE_BUILD_TYPE=${{ env.build_type }} -DENABLE_TESTS=OFF -DENABLE_UTILS=OFF -DENABLE_EXAMPLES=OFF -DENABLE_PULSAR=OFF
 
       # Compile the project using Ninja
       - name: Build with Ninja

.gitmodules

@@ -283,3 +283,6 @@
 [submodule "contrib/c-ares"]
 	path = contrib/c-ares
 	url = https://github.com/c-ares/c-ares.git
+[submodule "contrib/pulsar-cmake/pulsar-client-cpp"]
+	path = contrib/pulsar-cmake/pulsar-client-cpp
+	url = https://github.com/apache/pulsar-client-cpp.git

build.sh

@@ -4,6 +4,12 @@
 build_type="$1"
 build_type="${build_type:=Release}"
 
+is_not_darwin=1
+if [[ "$(uname)" == "Darwin" ]]; then
+    is_not_darwin=0
+    echo "=== This is Darwin ==="
+fi
+
 sanitizer=""
 if [ "$build_type" = "Debug" ]; then
     sanitizer="address"
@@ -74,5 +80,7 @@ cmake .. \
     -DENABLE_ENCRYPT_DECRYPT_FUNCS=ON \
     -DENABLE_DEBUG_FUNCS=ON \
     -DENABLE_URL_FUNCS=ON \
-    -DENABLE_AVRO=ON
+    -DENABLE_AVRO=ON \
+    -DENABLE_CURL=${is_not_darwin} \
+    -DENABLE_PULSAR=${is_not_darwin}
 

contrib/CMakeLists.txt

@@ -129,6 +129,7 @@ if (NOT OS_DARWIN)
     add_contrib (curl-cmake curl)
     add_contrib (azure-cmake azure)
     add_contrib (sentry-native-cmake sentry-native) # requires: curl
+    add_contrib (pulsar-cmake) # requires: curl
 endif()
 
 add_contrib (fmtlib-cmake fmtlib)

contrib/pulsar-cmake/.gitignore

@@ -0,0 +1,2 @@
+!generated/lib/PulsarApi.pb.h
+!generated/lib/PulsarApi.pb.cc

contrib/pulsar-cmake/CMakeLists.txt

@@ -0,0 +1,73 @@
+# Needed when using Apache Pulsar external stream
+option (ENABLE_PULSAR "Enable Pulsar" ${ENABLE_LIBRARIES})
+
+if (NOT ENABLE_PULSAR)
+    message(STATUS "Not using pulsar")
+    return()
+endif()
+
+if (NOT ENABLE_CURL)
+    message(FATAL_ERROR "Pulsar requires curl")
+endif()
+
+if (OS_DARWIN)
+    message(FATAL_ERROR "Pulsar is not supported on MacOS yet")
+endif()
+
+set(PULSAR_CLIENT_CPP_DIR "${CMAKE_CURRENT_LIST_DIR}/pulsar-client-cpp")
+set(PROTOC_PATH protoc)
+
+execute_process(COMMAND cat ${PULSAR_CLIENT_CPP_DIR}/version.txt OUTPUT_STRIP_TRAILING_WHITESPACE
+            OUTPUT_VARIABLE PULSAR_CLIENT_VERSION)
+message(STATUS "Pulsar Client version: ${PULSAR_CLIENT_VERSION}")
+
+execute_process(COMMAND ${PULSAR_CLIENT_CPP_DIR}/build-support/gen-pulsar-version-macro.py OUTPUT_STRIP_TRAILING_WHITESPACE
+        OUTPUT_VARIABLE PULSAR_CLIENT_VERSION_MACRO)
+message(STATUS "Pulsar Client version macro: ${PULSAR_CLIENT_VERSION_MACRO}")
+
+set(PVM_COMMENT "This is generated from Version.h.in by CMAKE. DO NOT EDIT DIRECTLY")
+configure_file(${PULSAR_CLIENT_CPP_DIR}/templates/Version.h.in ${PULSAR_CLIENT_CPP_DIR}/include/pulsar/Version.h @ONLY)
+
+set(LIB_AUTOGEN_DIR ${CMAKE_CURRENT_LIST_DIR}/generated/lib)
+set(PROTO_SOURCES
+    ${LIB_AUTOGEN_DIR}/PulsarApi.pb.cc
+    ${LIB_AUTOGEN_DIR}/PulsarApi.pb.h)
+
+file(GLOB PULSAR_SOURCES
+    ${PULSAR_CLIENT_CPP_DIR}/lib/*.cc
+    ${PULSAR_CLIENT_CPP_DIR}/lib/*.h
+    ${PULSAR_CLIENT_CPP_DIR}/lib/lz4/*.cc
+    ${PULSAR_CLIENT_CPP_DIR}/lib/lz4/*.h
+    ${PULSAR_CLIENT_CPP_DIR}/lib/checksum/*.cc
+    ${PULSAR_CLIENT_CPP_DIR}/lib/checksum/*.h
+    ${PULSAR_CLIENT_CPP_DIR}/lib/stats/*.cc
+    ${PULSAR_CLIENT_CPP_DIR}/lib/stats/*.h
+    ${PULSAR_CLIENT_CPP_DIR}/lib/c/*.cc
+    ${PULSAR_CLIENT_CPP_DIR}/lib/c/*.h
+    ${PULSAR_CLIENT_CPP_DIR}/lib/auth/*.cc
+    ${PULSAR_CLIENT_CPP_DIR}/lib/auth/*.h
+    ${PULSAR_CLIENT_CPP_DIR}/lib/auth/athenz/*.cc
+    ${PULSAR_CLIENT_CPP_DIR}/lib/auth/athenz/*.h)
+
+set(PULSAR_SOURCES ${PULSAR_SOURCES} ${PROTO_SOURCES})
+
+add_library(_pulsar ${PULSAR_SOURCES})
+add_library(ch_contrib::pulsar ALIAS _pulsar)
+set_target_properties (_pulsar PROPERTIES VERSION ${PULSAR_CLIENT_VERSION})
+
+target_include_directories(_pulsar SYSTEM PUBLIC "${PULSAR_CLIENT_CPP_DIR}/include")
+target_include_directories(_pulsar SYSTEM PUBLIC "${PULSAR_CLIENT_CPP_DIR}")
+target_include_directories(_pulsar SYSTEM PUBLIC "${LIB_AUTOGEN_DIR}")
+
+target_link_libraries(_pulsar PRIVATE boost::headers_only)
+target_link_libraries(_pulsar PRIVATE ch_contrib::curl)
+target_link_libraries(_pulsar
+  PRIVATE
+    ch_contrib::lz4
+    ch_contrib::zlib
+    ch_contrib::zstd
+    OpenSSL::Crypto
+    OpenSSL::SSL
+    ch_contrib::protobuf
+    ch_contrib::snappy
+)

contrib/pulsar-cmake/README.md

@@ -0,0 +1,3 @@
+## About the `generated` folder ##
+
+In order to build pulsar-client-cpp, it requires `protoc` to generate the C++ code for `PulsarApi.proto`, which just unnecessarily complicates the build tool chain (for us). And also, the bigger problem is that, currently, `proton` still uses a out-of-date protobuf library, thus it requires an old version of `protoc` to generate the code. So we simply just pre-generated the source code from the Protobuf file and hosted the code directly.