refactor(profiling): remove legacy Python exporter and enable libdatadog exporter by default (#13599)

Removes python exporter and enable libdatadog exporter by default. Since
stack v2 requires libdatadog exporter and it's the default mode, most
customers would not be affected by this.

We are dropping 32-bit Linux support for profiling, but this can be
added back once profiling depends on libdatadog via Rust bindings using
PyO3.

## Checklist
- [x] PR author has checked that all the criteria below are met
- The PR description includes an overview of the change
- The PR description articulates the motivation for the change
- The change includes tests OR the PR description describes a testing
strategy
- The PR description notes risks associated with the change, if any
- Newly-added code is easy to change
- The change follows the [library release note
guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html)
- The change includes or references documentation updates if necessary
- Backport labels are set (if
[applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting))

## Reviewer Checklist
- [x] Reviewer has checked that all the criteria below are met 
- Title is accurate
- All changes are related to the pull request's stated goal
- Avoids breaking
[API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces)
changes
- Testing strategy adequately addresses listed risks
- Newly-added code is easy to change
- Release note makes sense to a user of the library
- If necessary, author has acknowledged and discussed the performance
implications of this PR as reported in the benchmarks PR comment
- Backport labels are set in a manner that is consistent with the
[release branch maintenance
policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)

Author: taegyunkim

ddtrace/internal/datadog/profiling/ddup/__init__.py

@@ -1,7 +1,7 @@
-# This module supports an optional feature.  It may not even load on all platforms or configurations.
-# In ddtrace/settings/profiling.py, this module is imported and the is_available attribute is checked to determine
-# whether the feature is available. If not, then the feature is disabled and all downstream consumption is
-# suppressed.
+# In case when _ddup is not available, we set is_available attribute to False,
+# and failure_msg to the error message. This module is initially imported in
+# ddtrace/settings/profiling.py to determine if profiling can be run. If it
+# fails, we turn off profiling in ddtrace/settings/profiling.py
 is_available = False
 failure_msg = ""
 

ddtrace/profiling/bootstrap/sitecustomize.py

@@ -1,9 +1,17 @@
 # -*- encoding: utf-8 -*-
 """Bootstrapping code that is run when using `ddtrace.profiling.auto`."""
+
+import platform
+import sys
+
+from ddtrace.internal.logger import get_logger
 from ddtrace.profiling import bootstrap
 from ddtrace.profiling import profiler
 
 
+LOG = get_logger(__name__)
+
+
 def start_profiler():
     if hasattr(bootstrap, "profiler"):
         bootstrap.profiler.stop()
@@ -12,4 +20,12 @@ def start_profiler():
     bootstrap.profiler.start()
 
 
-start_profiler()
+if platform.system() == "Linux" and not (sys.maxsize > (1 << 32)):
+    LOG.error(
+        "The Datadog Profiler is not supported on 32-bit Linux systems. "
+        "To use the profiler, please upgrade to a 64-bit Linux system. "
+        "If you believe this is an error or need assistance, please report it at "
+        "https://github.com/DataDog/dd-trace-py/issues"
+    )
+else:
+    start_profiler()

ddtrace/profiling/collector/__init__.py

@@ -1,13 +1,8 @@
 # -*- encoding: utf-8 -*-
-import typing  # noqa:F401
-
 from ddtrace.internal import periodic
 from ddtrace.internal import service
 from ddtrace.settings.profiling import config
 
-from .. import event  # noqa:F401
-from ..recorder import Recorder
-
 
 class CollectorError(Exception):
     pass
@@ -20,16 +15,12 @@ class CollectorUnavailable(CollectorError):
 class Collector(service.Service):
     """A profile collector."""
 
-    def __init__(self, recorder: typing.Optional[Recorder] = None, *args, **kwargs):
+    def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.recorder = recorder
 
     @staticmethod
     def snapshot():
-        """Take a snapshot of collected data.
-
-        :return: A list of sample list to push in the recorder.
-        """
+        """Take a snapshot of collected data, to be exported."""
 
 
 class PeriodicCollector(Collector, periodic.PeriodicService):
@@ -38,18 +29,11 @@ class PeriodicCollector(Collector, periodic.PeriodicService):
     __slots__ = ()
 
     def periodic(self):
-        # type: (...) -> None
-        """Collect events and push them into the recorder."""
-        for events in self.collect():
-            if self.recorder:
-                self.recorder.push_events(events)
+        # This is to simply override periodic.PeriodicService.periodic()
+        self.collect()
 
     def collect(self):
-        # type: (...) -> typing.Iterable[typing.Iterable[event.Event]]
-        """Collect the actual data.
-
-        :return: A list of event list to push in the recorder.
-        """
+        """Collect the actual data."""
         raise NotImplementedError
 
 
@@ -77,7 +61,7 @@ def capture(self):
 
 
 class CaptureSamplerCollector(Collector):
-    def __init__(self, recorder: typing.Optional[Recorder] = None, capture_pct=config.capture_pct, *args, **kwargs):
-        super().__init__(recorder, *args, **kwargs)
+    def __init__(self, capture_pct=config.capture_pct, *args, **kwargs):
+        super().__init__(*args, **kwargs)
         self.capture_pct = capture_pct
         self._capture_sampler = CaptureSampler(self.capture_pct)

ddtrace/profiling/collector/_lock.py

@@ -13,50 +13,12 @@
 from ddtrace.internal.datadog.profiling import ddup
 from ddtrace.profiling import _threading
 from ddtrace.profiling import collector
-from ddtrace.profiling import event
 from ddtrace.profiling.collector import _task
 from ddtrace.profiling.collector import _traceback
-from ddtrace.profiling.recorder import Recorder
 from ddtrace.settings.profiling import config
 from ddtrace.trace import Tracer
 
 
-class LockEventBase(event.StackBasedEvent):
-    """Base Lock event."""
-
-    __slots__ = ("lock_name", "sampling_pct")
-
-    def __init__(self, lock_name="<unknown lock name>", sampling_pct=0, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.lock_name = lock_name
-        self.sampling_pct = sampling_pct
-
-
-class LockAcquireEvent(LockEventBase):
-    """A lock has been acquired."""
-
-    __slots__ = ("wait_time_ns",)
-
-    def __init__(self, wait_time_ns=0, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.wait_time_ns = wait_time_ns
-
-
-class LockReleaseEvent(LockEventBase):
-    """A lock has been released."""
-
-    __slots__ = ("locked_for_ns",)
-
-    def __init__(
-        self,
-        locked_for_ns=0,  # type: int
-        *args,  # type: typing.Any
-        **kwargs,  # type: typing.Any
-    ):
-        super().__init__(*args, **kwargs)
-        self.locked_for_ns: int = locked_for_ns
-
-
 def _current_thread():
     # type: (...) -> typing.Tuple[int, str]
     thread_id = _thread.get_ident()
@@ -78,26 +40,19 @@ def _current_thread():
 
 
 class _ProfiledLock(wrapt.ObjectProxy):
-    ACQUIRE_EVENT_CLASS = LockAcquireEvent
-    RELEASE_EVENT_CLASS = LockReleaseEvent
-
     def __init__(
         self,
         wrapped: typing.Any,
-        recorder: Recorder,
         tracer: typing.Optional[Tracer],
         max_nframes: int,
         capture_sampler: collector.CaptureSampler,
         endpoint_collection_enabled: bool,
-        export_libdd_enabled: bool,
     ) -> None:
         wrapt.ObjectProxy.__init__(self, wrapped)
-        self._self_recorder = recorder
         self._self_tracer = tracer
         self._self_max_nframes = max_nframes
         self._self_capture_sampler = capture_sampler
         self._self_endpoint_collection_enabled = endpoint_collection_enabled
-        self._self_export_libdd_enabled = export_libdd_enabled
         frame = sys._getframe(2 if WRAPT_C_EXT else 3)
         code = frame.f_code
         self._self_init_loc = "%s:%d" % (os.path.basename(code.co_filename), frame.f_lineno)
@@ -131,41 +86,23 @@ def _acquire(self, inner_func, *args, **kwargs):
                 else:
                     frame = task_frame
 
-                frames, nframes = _traceback.pyframe_to_frames(frame, self._self_max_nframes)
+                frames, _ = _traceback.pyframe_to_frames(frame, self._self_max_nframes)
 
-                if self._self_export_libdd_enabled:
-                    thread_native_id = _threading.get_thread_native_id(thread_id)
+                thread_native_id = _threading.get_thread_native_id(thread_id)
 
-                    handle = ddup.SampleHandle()
-                    handle.push_monotonic_ns(end)
-                    handle.push_lock_name(lock_name)
-                    handle.push_acquire(end - start, 1)  # AFAICT, capture_pct does not adjust anything here
-                    handle.push_threadinfo(thread_id, thread_native_id, thread_name)
-                    handle.push_task_id(task_id)
-                    handle.push_task_name(task_name)
+                handle = ddup.SampleHandle()
+                handle.push_monotonic_ns(end)
+                handle.push_lock_name(lock_name)
+                handle.push_acquire(end - start, 1)  # AFAICT, capture_pct does not adjust anything here
+                handle.push_threadinfo(thread_id, thread_native_id, thread_name)
+                handle.push_task_id(task_id)
+                handle.push_task_name(task_name)
 
-                    if self._self_tracer is not None:
-                        handle.push_span(self._self_tracer.current_span())
-                    for frame in frames:
-                        handle.push_frame(frame.function_name, frame.file_name, 0, frame.lineno)
-                    handle.flush_sample()
-                else:
-                    event = self.ACQUIRE_EVENT_CLASS(
-                        lock_name=lock_name,
-                        frames=frames,
-                        nframes=nframes,
-                        thread_id=thread_id,
-                        thread_name=thread_name,
-                        task_id=task_id,
-                        task_name=task_name,
-                        wait_time_ns=end - start,
-                        sampling_pct=self._self_capture_sampler.capture_pct,
-                    )
-
-                    if self._self_tracer is not None:
-                        event.set_trace_info(self._self_tracer.current_span(), self._self_endpoint_collection_enabled)
-
-                    self._self_recorder.push_event(event)
+                if self._self_tracer is not None:
+                    handle.push_span(self._self_tracer.current_span())
+                for frame in frames:
+                    handle.push_frame(frame.function_name, frame.file_name, 0, frame.lineno)
+                handle.flush_sample()
             except Exception:
                 pass  # nosec
 
@@ -207,41 +144,23 @@ def _release(self, inner_func, *args, **kwargs):
                 else:
                     frame = task_frame
 
-                frames, nframes = _traceback.pyframe_to_frames(frame, self._self_max_nframes)
+                frames, _ = _traceback.pyframe_to_frames(frame, self._self_max_nframes)
 
-                if self._self_export_libdd_enabled:
-                    thread_native_id = _threading.get_thread_native_id(thread_id)
+                thread_native_id = _threading.get_thread_native_id(thread_id)
 
-                    handle = ddup.SampleHandle()
-                    handle.push_monotonic_ns(end)
-                    handle.push_lock_name(lock_name)
-                    handle.push_release(end - start, 1)  # AFAICT, capture_pct does not adjust anything here
-                    handle.push_threadinfo(thread_id, thread_native_id, thread_name)
-                    handle.push_task_id(task_id)
-                    handle.push_task_name(task_name)
+                handle = ddup.SampleHandle()
+                handle.push_monotonic_ns(end)
+                handle.push_lock_name(lock_name)
+                handle.push_release(end - start, 1)  # AFAICT, capture_pct does not adjust anything here
+                handle.push_threadinfo(thread_id, thread_native_id, thread_name)
+                handle.push_task_id(task_id)
+                handle.push_task_name(task_name)
 
-                    if self._self_tracer is not None:
-                        handle.push_span(self._self_tracer.current_span())
-                    for frame in frames:
-                        handle.push_frame(frame.function_name, frame.file_name, 0, frame.lineno)
-                    handle.flush_sample()
-                else:
-                    event = self.RELEASE_EVENT_CLASS(
-                        lock_name=lock_name,
-                        frames=frames,
-                        nframes=nframes,
-                        thread_id=thread_id,
-                        thread_name=thread_name,
-                        task_id=task_id,
-                        task_name=task_name,
-                        locked_for_ns=end - start,
-                        sampling_pct=self._self_capture_sampler.capture_pct,
-                    )
-
-                    if self._self_tracer is not None:
-                        event.set_trace_info(self._self_tracer.current_span(), self._self_endpoint_collection_enabled)
-
-                    self._self_recorder.push_event(event)
+                if self._self_tracer is not None:
+                    handle.push_span(self._self_tracer.current_span())
+                for frame in frames:
+                    handle.push_frame(frame.function_name, frame.file_name, 0, frame.lineno)
+                handle.flush_sample()
 
     def release(self, *args, **kwargs):
         return self._release(self.__wrapped__.release, *args, **kwargs)
@@ -313,23 +232,17 @@ class LockCollector(collector.CaptureSamplerCollector):
 
     def __init__(
         self,
-        recorder: typing.Optional[Recorder] = None,
         nframes=config.max_frames,
         endpoint_collection_enabled=config.endpoint_collection,
-        export_libdd_enabled=config.export.libdd_enabled,
         tracer=None,
         *args,
         **kwargs,
     ):
-        super().__init__(recorder, *args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.nframes = nframes
         self.endpoint_collection_enabled = endpoint_collection_enabled
-        self.export_libdd_enabled = export_libdd_enabled
         self.tracer = tracer
         self._original = None
-        # Check if libdd is available, if not, disable the feature
-        if self.export_libdd_enabled and not ddup.is_available:
-            self.export_libdd_enabled = False
 
     @abc.abstractmethod
     def _get_patch_target(self):
@@ -367,12 +280,10 @@ def _allocate_lock(wrapped, instance, args, kwargs):
             lock = wrapped(*args, **kwargs)
             return self.PROFILED_LOCK_CLASS(
                 lock,
-                self.recorder,
                 self.tracer,
                 self.nframes,
                 self._capture_sampler,
                 self.endpoint_collection_enabled,
-                self.export_libdd_enabled,
             )
 
         self._set_patch_target(FunctionWrapper(self._original, _allocate_lock))

ddtrace/profiling/collector/_task.pyx

@@ -1,4 +1,3 @@
-import sys
 from types import ModuleType
 import weakref
 
@@ -23,7 +22,7 @@ def install_greenlet_tracer(gevent):
         from greenlet import settrace
     except ImportError:
         # We don't seem to have the required dependencies.
-        return 
+        return
 
     class DDGreenletTracer(object):
         def __init__(self, gevent):

ddtrace/profiling/collector/asyncio.py

@@ -4,21 +4,8 @@
 from . import _lock
 
 
-class AsyncioLockAcquireEvent(_lock.LockAcquireEvent):
-    """An asyncio.Lock has been acquired."""
-
-    __slots__ = ()
-
-
-class AsyncioLockReleaseEvent(_lock.LockReleaseEvent):
-    """An asyncio.Lock has been released."""
-
-    __slots__ = ()
-
-
 class _ProfiledAsyncioLock(_lock._ProfiledLock):
-    ACQUIRE_EVENT_CLASS = AsyncioLockAcquireEvent
-    RELEASE_EVENT_CLASS = AsyncioLockReleaseEvent
+    pass
 
 
 class AsyncioLockCollector(_lock.LockCollector):