feat(compute_ctl): CPU Continuous Profiling support

Cargo.toml

@@ -130,6 +130,7 @@ jemalloc_pprof = { version = "0.7", features = ["symbolize", "flamegraph"] }
 jsonwebtoken = "9"
 lasso = "0.7"
 libc = "0.2"
+libproc = "0.14"
 md5 = "0.7.0"
 measured = { version = "0.0.22", features=["lasso"] }
 measured-process = { version = "0.0.22" }
@@ -278,6 +279,7 @@ safekeeper_api = { version = "0.1", path = "./libs/safekeeper_api" }
 safekeeper_client = { path = "./safekeeper/client" }
 storage_broker = { version = "0.1", path = "./storage_broker/" } # Note: main broker code is inside the binary crate, so linking with the library shouldn't be heavy.
 storage_controller_client = { path = "./storage_controller/client" }
+tempfile = "3"
 tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" }
 tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" }
 utils = { version = "0.1", path = "./libs/utils/" }

Dockerfile

@@ -109,6 +109,8 @@ RUN set -e \
         libreadline-dev \
         libseccomp-dev \
         ca-certificates \
+        bpfcc-tools \
+        sudo \
         openssl \
         unzip \
         curl \

build-tools.Dockerfile

@@ -61,6 +61,9 @@ RUN if [ "${DEBIAN_VERSION}" = "bookworm" ]; then \
         libpq5 \
         libpq-dev \
         libzstd-dev \
+        linux-perf \
+        bpfcc-tools \
+        linux-headers-$(case "$(uname -m)" in x86_64) echo amd64;; aarch64) echo arm64;; esac) \
         postgresql-16 \
         postgresql-server-dev-16 \
         postgresql-common  \
@@ -105,15 +108,21 @@ RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
 #
 # 'gdb' is included so that we get backtraces of core dumps produced in
 # regression tests
-RUN set -e \
+RUN set -ex \
+    && KERNEL_VERSION="$(uname -r | cut -d'-' -f1 | sed 's/\.0$//')" \
+    && echo KERNEL_VERSION=${KERNEL_VERSION} >> /etc/environment \
+    && KERNEL_ARCH=$(uname -m | awk '{ if ($1 ~ /^(x86_64|i[3-6]86)$/) print "x86"; else if ($1 ~ /^(aarch64|arm.*)$/) print "aarch"; else print $1 }') \
+    && echo KERNEL_ARCH=${KERNEL_ARCH} >> /etc/environment \
     && apt update \
     && apt install -y \
         autoconf \
         automake \
+        bc \
         bison \
         build-essential \
         ca-certificates \
         cmake \
+        cpio \
         curl \
         flex \
         gdb \
@@ -122,8 +131,10 @@ RUN set -e \
         gzip \
         jq \
         jsonnet \
+        kmod \
         libcurl4-openssl-dev \
         libbz2-dev \
+        libelf-dev \
         libffi-dev \
         liblzma-dev \
         libncurses5-dev \
@@ -137,19 +148,51 @@ RUN set -e \
         libxml2-dev \
         libxmlsec1-dev \
         libxxhash-dev \
+        linux-perf \
+        bpfcc-tools \
+        libbpfcc \
+        libbpfcc-dev \
+        linux-headers-$(case "$(uname -m)" in x86_64) echo amd64;; aarch64) echo arm64;; esac) \
         lsof \
         make \
         netcat-openbsd \
         net-tools \
         openssh-client \
         parallel \
         pkg-config \
+        rsync \
+        sudo \
         unzip \
         wget \
         xz-utils \
         zlib1g-dev \
         zstd \
-    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+    && git clone --depth 1 --branch v${KERNEL_VERSION} https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git \
+    && cd linux \
+    && make mrproper \
+    && make defconfig \
+    && scripts/config --module CONFIG_IKHEADERS \
+    && scripts/config --enable CONFIG_MODULE_COMPRESS \
+    && scripts/config --disable CONFIG_MODULE_COMPRESS_GZIP \
+    && scripts/config --enable CONFIG_MODULE_COMPRESS_ZSTD \
+    && make olddefconfig \
+    && make WERROR=0 NO_WERROR=1 modules_prepare -j \
+    && make WERROR=0 NO_WERROR=1 -j 10 \
+    && make WERROR=0 NO_WERROR=1 modules -j10 \
+    && mkdir -p /lib/modules/$(uname -r)/build \
+    && mkdir -p /lib/modules/$(uname -r)/kernel/kernel \
+    && cp -a include arch/${KERNEL_ARCH}/include scripts Module.symvers .config Makefile /lib/modules/$(uname -r)/build/ \
+    && make headers_install INSTALL_HDR_PATH=/lib/modules/$(uname -r)/build \
+    && mkdir -p /lib/modules/$(uname -r)/build/arch/${KERNEL_ARCH}/include \
+    && rsync -a arch/${KERNEL_ARCH}/include /lib/modules/$(uname -r)/build/arch/${KERNEL_ARCH}/ \
+    && zstd -19 ./kernel/kheaders.ko -o ./kernel/kheaders.ko.zst \
+    && cp -a kernel/kheaders.ko.zst /lib/modules/$(uname -r)/kernel/kernel \
+    && find /lib/modules/ -iname "*rwonce.h*" \
+    && mkdir -p /virtual/include \
+    && cp -a /lib/modules/$(uname -r)/build/include /virtual/include/ \
+    && execsnoop-bpfcc \
+    && rm -rf linux
 
 # sql_exporter
 
@@ -198,6 +241,8 @@ RUN curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /
 # Configure sudo & docker
 RUN usermod -aG sudo nonroot && \
     echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    mkdir -p /etc/sudoers.d && \
+    echo 'nonroot ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/nonroot && \
     usermod -aG docker nonroot
 
 # AWS CLI

compute/vm-image-spec-bookworm.yaml

@@ -65,7 +65,7 @@ files:
       # regardless of hostname (ALL)
       #
       # Also allow it to shut down the VM. The fast_import job does that when it's finished.
-      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff, /usr/sbin/rsyslogd
+      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff, /usr/sbin/rsyslogd, /neonvm/tools/bin/perf, /usr/sbin/profile-bpfcc
   - filename: cgconfig.conf
     content: |
       # Configuration for cgroups in VM compute nodes
@@ -152,6 +152,8 @@ merge: |
   RUN set -e \
       && chmod 0644 /etc/cgconfig.conf
 
+  ENV PERF_BINARY_PATH=/neonvm/tools/bin/perf
+
 
   COPY compute_rsyslog.conf /etc/compute_rsyslog.conf
   RUN chmod 0666 /etc/compute_rsyslog.conf

compute/vm-image-spec-bullseye.yaml

@@ -65,7 +65,7 @@ files:
       # regardless of hostname (ALL)
       #
       # Also allow it to shut down the VM. The fast_import job does that when it's finished.
-      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff, /usr/sbin/rsyslogd
+      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff, /usr/sbin/rsyslogd, /neonvm/tools/bin/perf, /usr/sbin/profile-bpfcc
   - filename: cgconfig.conf
     content: |
       # Configuration for cgroups in VM compute nodes
@@ -148,6 +148,8 @@ merge: |
   RUN set -e \
       && chmod 0644 /etc/cgconfig.conf
 
+  ENV PERF_BINARY_PATH=/neonvm/tools/bin/perf
+
   COPY compute_rsyslog.conf /etc/compute_rsyslog.conf
   RUN chmod 0666 /etc/compute_rsyslog.conf
   RUN mkdir /var/log/rsyslog && chown -R postgres /var/log/rsyslog

compute_tools/Cargo.toml

@@ -31,6 +31,7 @@ hostname-validator = "1.1"
 indexmap.workspace = true
 itertools.workspace = true
 jsonwebtoken.workspace = true
+libproc.workspace = true
 metrics.workspace = true
 nix.workspace = true
 notify.workspace = true
@@ -49,6 +50,7 @@ serde_with.workspace = true
 serde_json.workspace = true
 signal-hook.workspace = true
 tar.workspace = true
+tempfile.workspace = true
 tower.workspace = true
 tower-http.workspace = true
 tokio = { workspace = true, features = ["rt", "rt-multi-thread"] }
@@ -78,3 +80,10 @@ zstd = "0.13"
 bytes = "1.0"
 rust-ini = "0.20.0"
 rlimit = "0.10.1"
+
+inferno = { version = "0.12", default-features = false, features = [
+    "multithreaded",
+    "nameattr",
+] }
+pprof = { version = "0.15", features = ["protobuf-codec", "flamegraph"] }
+prost = "0.12"

compute_tools/src/compute.rs

@@ -371,7 +371,9 @@ fn maybe_cgexec(cmd: &str) -> Command {
     }
 }
 
-struct PostgresHandle {
+/// A handle to the Postgres process that is running in the compute
+/// node.
+pub struct PostgresHandle {
     postgres: std::process::Child,
     log_collector: JoinHandle<Result<()>>,
 }

compute_tools/src/http/routes/mod.rs

@@ -15,6 +15,7 @@ pub(in crate::http) mod lfc;
 pub(in crate::http) mod metrics;
 pub(in crate::http) mod metrics_json;
 pub(in crate::http) mod promote;
+pub(in crate::http) mod profile;
 pub(in crate::http) mod status;
 pub(in crate::http) mod terminate;