Merge profiling and libnative libraries.

build_libnative.py

@@ -0,0 +1,76 @@
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+def is_installed(bin_file):
+    for path in os.environ.get("PATH", "").split(os.pathsep):
+        if os.path.isfile(os.path.join(path, bin_file)):
+            return True
+
+    return False
+
+def install_dedup_headers():
+    if not is_installed("dedup_headers"):
+        subprocess.run(
+            ["cargo", "install", "--git", "https://github.com/DataDog/libdatadog", "--bin", "dedup_headers", "tools"],
+            check=True,
+            )
+
+
+def build_crate(crate_dir: Path, release: bool, features: list = None):
+    env = os.environ.copy()
+    abs_dir = crate_dir.absolute()
+    env["CARGO_TARGET_DIR"] = str(abs_dir / "target")
+
+    build_cmd = ["cargo", "build"]
+
+    if release:
+        build_cmd.append("--release")
+
+    if features:
+        cmd_features = ', '.join(features)
+        build_cmd += ["--features", cmd_features]
+
+    subprocess.run(
+            build_cmd,
+            cwd=str(crate_dir),
+            check=True,
+            env=env,
+            )
+
+    if 'profiling' in features:
+        install_dedup_headers()
+
+        subprocess.run(
+            ["dedup_headers", "common.h", "crashtracker.h", "profiling.h"],
+            cwd=str(abs_dir / "target" / "include" / "datadog"),
+            check=True,
+        )
+
+def clean_crate(crate_dir: Path):
+    target_dir = crate_dir.absolute() / "target"
+    if target_dir.exists():
+        subprocess.run(
+            ["cargo", "clean"],
+            cwd=str(crate_dir),
+            check=True,
+        )
+
+def main():
+    try:
+        parser = argparse.ArgumentParser(description="Build Rust module")
+        parser.add_argument("--crate", required=True, help="Rust crate location")
+        parser.add_argument("--release", action="store_true", help="Release profile")
+        parser.add_argument("--features", nargs='*', help="List of features")
+
+        args = parser.parse_args()
+
+        build_crate(Path(args.crate), args.release, args.features)
+    except Exception as e:
+        print(f"Error: {e}")
+        sys.exit(1)
+
+if __name__ == "__main__":
+    main()

ddtrace/internal/datadog/profiling/build_standalone.sh

@@ -365,6 +365,12 @@ add_target() {
   esac
 }
 
+#Build rust dependencies
+build_rust() {
+    echo "Building Rust dependencies"
+    python3 build_libnative.py --crate ./src/native --release --features profiling
+}
+
 
 ### ENTRYPOINT
 # Check for basic input validity
@@ -383,6 +389,8 @@ print_cmake_args
 
 print_ctest_args
 
+build_rust
+
 # Run cmake
 for target in "${targets[@]}"; do
   run_cmake $target

ddtrace/internal/datadog/profiling/cmake/FindLibNative.cmake

@@ -0,0 +1,20 @@
+set(SOURCE_INCLUDE_DIR ${CMAKE_SOURCE_DIR}/../../../../../src/native/target/include)
+set(SOURCE_LIB_DIR ${CMAKE_SOURCE_DIR}/../../../../../src/native/target/release)
+
+set(DEST_LIB_DIR ${CMAKE_CURRENT_BINARY_DIR})
+set(DEST_INCLUDE_DIR ${DEST_LIB_DIR}/include)
+set(DEST_LIB_OUTPUT_DIR ${DEST_LIB_DIR})
+
+file(COPY ${SOURCE_INCLUDE_DIR} DESTINATION ${DEST_LIB_DIR})
+
+file(GLOB LIB_FILES "${SOURCE_LIB_DIR}/*.so" "${SOURCE_LIB_DIR}/*.lib" "${SOURCE_LIB_DIR}/*.dll")
+file(COPY ${LIB_FILES} DESTINATION ${DEST_LIB_OUTPUT_DIR})
+
+# Add imported library target
+add_library(_native SHARED IMPORTED)
+
+set_target_properties(_native PROPERTIES
+    IMPORTED_LOCATION ${DEST_LIB_OUTPUT_DIR}/lib_native.so
+    INTERFACE_INCLUDE_DIRECTORIES ${DEST_INCLUDE_DIR}
+)
+

ddtrace/internal/datadog/profiling/crashtracker/CMakeLists.txt

@@ -81,7 +81,7 @@ if(LIB_INSTALL_DIR)
 endif()
 
 # Crashtracker receiver binary
-add_executable(crashtracker_exe src/crashtracker.cpp)
+add_executable(crashtracker_exe src/crashtracker.cpp python_runtime_stub.c)
 target_include_directories(crashtracker_exe PRIVATE .. ${Datadog_INCLUDE_DIRS})
 
 # The CRASHTRACKER_EXE_TARGET_NAME should have been set by dd_wrapper

ddtrace/internal/datadog/profiling/crashtracker/python_runtime_stub.c

@@ -0,0 +1,81 @@
+__attribute__((weak)) void* PyObject_Repr = 0;
+__attribute__((weak)) void* PyExc_ValueError = 0;
+__attribute__((weak)) void* PyErr_GetRaisedException = 0;
+__attribute__((weak)) void* PyLong_FromSsize_t = 0;
+__attribute__((weak)) void* PyObject_GenericGetDict = 0;
+__attribute__((weak)) void* PyType_GetName = 0;
+__attribute__((weak)) void* PyBool_Type = 0;
+__attribute__((weak)) void* PyObject_GetAttr = 0;
+__attribute__((weak)) void* _Py_TrueStruct = 0;
+__attribute__((weak)) void* PyObject_Str = 0;
+__attribute__((weak)) void* PyType_FromSpec = 0;
+__attribute__((weak)) void* PyType_IsSubtype = 0;
+__attribute__((weak)) void* PyDict_SetItem = 0;
+__attribute__((weak)) void* PyTuple_New = 0;
+__attribute__((weak)) void* PyObject_SetAttr = 0;
+__attribute__((weak)) void* PyEval_RestoreThread = 0;
+__attribute__((weak)) void* _Py_NoneStruct = 0;
+__attribute__((weak)) void* PyExc_AttributeError = 0;
+__attribute__((weak)) void* PyException_SetTraceback = 0;
+__attribute__((weak)) void* PyList_Append = 0;
+__attribute__((weak)) void* PyInterpreterState_GetID = 0;
+__attribute__((weak)) void* PyBytes_FromStringAndSize = 0;
+__attribute__((weak)) void* PyUnicode_EqualToUTF8AndSize = 0;
+__attribute__((weak)) void* PyDict_Next = 0;
+__attribute__((weak)) void* PyException_GetTraceback = 0;
+__attribute__((weak)) void* PyErr_SetObject = 0;
+__attribute__((weak)) void* PyType_GenericAlloc = 0;
+__attribute__((weak)) void* PyType_GetModuleName = 0;
+__attribute__((weak)) void* PyFloat_FromDouble = 0;
+__attribute__((weak)) void* PyFloat_AsDouble = 0;
+__attribute__((weak)) void* _Py_Dealloc = 0;
+__attribute__((weak)) void* PyList_New = 0;
+__attribute__((weak)) void* PyType_GetQualName = 0;
+__attribute__((weak)) void* PyBytes_Size = 0;
+__attribute__((weak)) void* PyObject_DelItem = 0;
+__attribute__((weak)) void* PyErr_PrintEx = 0;
+__attribute__((weak)) void* PyDict_New = 0;
+__attribute__((weak)) void* PyErr_SetString = 0;
+__attribute__((weak)) void* PyExc_Exception = 0;
+__attribute__((weak)) void* PyObject_SetItem = 0;
+__attribute__((weak)) void* PyUnicode_InternInPlace = 0;
+__attribute__((weak)) void* PyErr_SetRaisedException = 0;
+__attribute__((weak)) void* PyObject_SetAttrString = 0;
+__attribute__((weak)) void* PyGILState_Release = 0;
+__attribute__((weak)) void* PyExc_RuntimeError = 0;
+__attribute__((weak)) void* PyExc_BaseException = 0;
+__attribute__((weak)) void* PyBytes_AsString = 0;
+__attribute__((weak)) void* PyObject_GenericSetDict = 0;
+__attribute__((weak)) void* PyBaseObject_Type = 0;
+__attribute__((weak)) void* PyObject_GetItem = 0;
+__attribute__((weak)) void* PyExc_TypeError = 0;
+__attribute__((weak)) void* PyEval_SaveThread = 0;
+__attribute__((weak)) void* PyObject_GC_UnTrack = 0;
+__attribute__((weak)) void* PyErr_WriteUnraisable = 0;
+__attribute__((weak)) void* PyModule_Create2 = 0;
+__attribute__((weak)) void* PyTraceBack_Print = 0;
+__attribute__((weak)) void* PyUnicode_AsEncodedString = 0;
+__attribute__((weak)) void* PyErr_NewExceptionWithDoc = 0;
+__attribute__((weak)) void* PyFloat_Type = 0;
+__attribute__((weak)) void* PyObject_CallNoArgs = 0;
+__attribute__((weak)) void* PyExc_SystemError = 0;
+__attribute__((weak)) void* PyExc_ImportError = 0;
+__attribute__((weak)) void* PyInterpreterState_Get = 0;
+__attribute__((weak)) void* PyUnicode_AsUTF8AndSize = 0;
+__attribute__((weak)) void* PyGILState_Ensure = 0;
+__attribute__((weak)) void* PyException_GetCause = 0;
+__attribute__((weak)) void* PyCMethod_New = 0;
+__attribute__((weak)) void* PyErr_Print = 0;
+__attribute__((weak)) void* PyImport_Import = 0;
+__attribute__((weak)) void* PyException_SetCause = 0;
+__attribute__((weak)) void* PyErr_GivenExceptionMatches = 0;
+__attribute__((weak)) void* Py_IsInitialized = 0;
+__attribute__((weak)) void* PyUnicode_FromStringAndSize = 0;
+__attribute__((weak)) void* PyErr_NormalizeException = 0;
+__attribute__((weak)) void* PyErr_Restore = 0;
+__attribute__((weak)) void* PyErr_Fetch = 0;
+__attribute__((weak)) void* PyObject_CallObject = 0;
+__attribute__((weak)) void* PyMem_Malloc = 0;
+__attribute__((weak)) void* PyCFunction_NewEx = 0;
+__attribute__((weak)) void* PyObject_Free = 0;
+

ddtrace/internal/datadog/profiling/dd_wrapper/CMakeLists.txt

@@ -24,7 +24,8 @@ include(FindInfer)
 include(CheckSymbolExists)
 
 # Load libdatadog
-include(FindLibdatadog)
+#include(FindLibdatadog)
+find_package(LibNative)
 
 # Set verbose mode so compiler and args are shown
 set(CMAKE_VERBOSE_MAKEFILE ON)
@@ -82,9 +83,9 @@ add_ddup_config(dd_wrapper)
 target_include_directories(dd_wrapper PRIVATE include ${Datadog_INCLUDE_DIRS})
 
 if(WIN32)
-    target_link_libraries(dd_wrapper PRIVATE Datadog::Profiling)
+    target_link_libraries(dd_wrapper PRIVATE _native)
 else()
-    target_link_libraries(dd_wrapper PRIVATE Datadog::Profiling Threads::Threads)
+    target_link_libraries(dd_wrapper PRIVATE _native Threads::Threads)
 endif()
 
 # Figure out the suffix.  Try to approximate the cpython way of doing things.

ddtrace/internal/datadog/profiling/dd_wrapper/test/CMakeLists.txt

@@ -1,3 +1,4 @@
+include(FetchContent)
 FetchContent_Declare(
     googletest
     GIT_REPOSITORY https://github.com/google/googletest.git
@@ -12,6 +13,8 @@ FetchContent_MakeAvailable(googletest)
 include(GoogleTest)
 include(AnalysisFunc)
 
+find_package(Python3 COMPONENTS Interpreter Development)
+
 if(DO_VALGRIND)
     find_program(
         VALGRIND_EXECUTABLE
@@ -33,8 +36,10 @@ endif()
 function(dd_wrapper_add_test name)
     add_executable(${name} ${ARGN})
     target_include_directories(${name} PRIVATE ../include)
-    target_link_libraries(${name} PRIVATE gmock gtest_main dd_wrapper)
+    #target_link_libraries(${name} PRIVATE ${Python3_LIBRARIES})
+    target_link_libraries(${name} PRIVATE gmock gtest_main dd_wrapper ${Python3_LIBRARIES})
     add_ddup_config(${name})
+    target_link_options(${name} PRIVATE -Wl,--no-as-needed)
 
     gtest_discover_tests(
         ${name}

ddtrace/internal/datadog/profiling/ddup/CMakeLists.txt

@@ -73,7 +73,9 @@ if(APPLE)
 elseif(UNIX)
     set_target_properties(${EXTENSION_NAME} PROPERTIES INSTALL_RPATH "$ORIGIN/..")
 endif()
-target_include_directories(${EXTENSION_NAME} PRIVATE ../dd_wrapper/include ${Datadog_INCLUDE_DIRS}
+target_include_directories(${EXTENSION_NAME} PRIVATE ../dd_wrapper/include
+                                                     ../../../../../src/native/target/include/
+                                                     ${Datadog_INCLUDE_DIRS}
                                                      ${Python3_INCLUDE_DIRS})
 
 target_link_libraries(${EXTENSION_NAME} PRIVATE dd_wrapper)

setup.py

@@ -2,6 +2,7 @@
 import fnmatch
 import hashlib
 import os
+from os.path import exists, isfile
 import platform
 import re
 import shutil
@@ -42,6 +43,11 @@
 from urllib.error import HTTPError
 from urllib.request import urlretrieve
 
+# workaround for ModuleNotFound.
+sys.path.insert(0, str(Path(__file__).parent.resolve()))
+
+from build_libnative import build_crate, clean_crate
+
 
 HERE = Path(__file__).resolve().parent
 
@@ -74,6 +80,7 @@
 DDUP_DIR = HERE / "ddtrace" / "internal" / "datadog" / "profiling" / "ddup"
 CRASHTRACKER_DIR = HERE / "ddtrace" / "internal" / "datadog" / "profiling" / "crashtracker"
 STACK_V2_DIR = HERE / "ddtrace" / "internal" / "datadog" / "profiling" / "stack_v2"
+NATIVE_CRATE = HERE / "src" / "native"
 
 BUILD_PROFILING_NATIVE_TESTS = os.getenv("DD_PROFILING_NATIVE_TESTS", "0").lower() in ("1", "yes", "on", "true")
 
@@ -306,12 +313,27 @@ def remove_artifacts():
         shutil.rmtree(LIBDDWAF_DOWNLOAD_DIR, True)
         shutil.rmtree(IAST_DIR / "*.so", True)
 
+    @staticmethod
+    def remove_rust():
+        clean_crate(NATIVE_CRATE)
+
     def run(self):
+        CleanLibraries.remove_rust()
         CleanLibraries.remove_artifacts()
         CleanCommand.run(self)
 
 
-class CMakeBuild(build_ext):
+class CustomBuildExt(build_ext):
+    def run(self):
+        self.build_rust()
+        super().run()
+        for ext in self.extensions:
+            self.build_extension(ext)
+
+    def build_rust(self):
+
+        build_crate(NATIVE_CRATE, True, native_features)
+
     @staticmethod
     def try_strip_symbols(so_file):
         if CURRENT_OS == "Linux" and shutil.which("strip") is not None:
@@ -605,6 +627,7 @@ def get_exts_for(name):
 
 
 if not IS_PYSTON:
+    native_features = []
     ext_modules = [
         Extension(
             "ddtrace.profiling.collector._memalloc",
@@ -662,6 +685,7 @@ def get_exts_for(name):
         )
 
     if (CURRENT_OS in ("Linux", "Darwin") and is_64_bit_python()) or CURRENT_OS == "Windows":
+        native_features.append("profiling")
         ext_modules.append(
             CMakeExtension(
                 "ddtrace.internal.datadog.profiling.ddup._ddup",
@@ -690,6 +714,7 @@ def get_exts_for(name):
 
 else:
     ext_modules = []
+    native_feautes = []
 
 interpose_sccache()
 setup(
@@ -709,7 +734,7 @@ def get_exts_for(name):
     # enum34 is an enum backport for earlier versions of python
     # funcsigs backport required for vendored debtcollector
     cmdclass={
-        "build_ext": CMakeBuild,
+        "build_ext": CustomBuildExt,
         "build_py": LibraryDownloader,
         "build_rust": build_rust,
         "clean": CleanLibraries,
@@ -780,15 +805,15 @@ def get_exts_for(name):
         compiler_directives={"language_level": "3"},
     )
     + filter_extensions(get_exts_for("psutil")),
-    rust_extensions=filter_extensions(
-        [
-            RustExtension(
-                "ddtrace.internal.native._native",
-                path="src/native/Cargo.toml",
-                py_limited_api="auto",
-                binding=Binding.PyO3,
-                debug=os.getenv("_DD_RUSTC_DEBUG") == "1",
-            ),
-        ]
-    ),
+    # rust_extensions=filter_extensions(
+    #     [
+    #         RustExtension(
+    #             "ddtrace.internal.native._native",
+    #             path="src/native/Cargo.toml",
+    #             py_limited_api="auto",
+    #             binding=Binding.PyO3,
+    #             debug=os.getenv("_DD_RUSTC_DEBUG") == "1",
+    #         ),
+    #     ]
+    # ),
 )

src/native/.cargo/config.toml

@@ -0,0 +1,8 @@
+[build]
+rustflags = ["-C", "relocation-model=pic"]
+
+[target.'cfg(target_os = "windows")']
+rustflags = ["-C", "relocation-model=pic", "-C", "target-feature=+crt-static"]
+
+[target.'cfg(all(target_os = "linux", target_env = "musl"))']
+rustflags = ["-C", "relocation-model=pic", "-C", "target-feature=+crt-static"]