From 3ff88b9184dc5f8fc9323c4b7792b2eb0b1890e3 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 10 Feb 2025 10:27:13 +0000 Subject: [PATCH 1/3] Add bpftrace allocations script Motivation: When debugging allocations it is frequently helpful to be able to use bpftrace instead of the more limited heaptrack. Modifications: Offer an equivalent of malloc-aggregation.d for bpftrace. Result: Allocation recording on Linux is available. --- dev/malloc-aggregation.bt | 51 +++++++++++++++++++++++++++++++++++ docs/debugging-allocations.md | 22 ++++++++++----- 2 files changed, 67 insertions(+), 6 deletions(-) create mode 100755 dev/malloc-aggregation.bt diff --git a/dev/malloc-aggregation.bt b/dev/malloc-aggregation.bt new file mode 100755 index 00000000000..9404326f4d1 --- /dev/null +++ b/dev/malloc-aggregation.bt @@ -0,0 +1,51 @@ +#!/usr/bin/env bpftrace +/*===----------------------------------------------------------------------===* + * + * This source file is part of the SwiftNIO open source project + * + * Copyright (c) 2017-2025 Apple Inc. and the SwiftNIO project authors + * Licensed under Apache License v2.0 + * + * See LICENSE.txt for license information + * See CONTRIBUTORS.txt for the list of SwiftNIO project authors + * + * SPDX-License-Identifier: Apache-2.0 + * + *===----------------------------------------------------------------------===*/ + +/* + * Example invocation: + * sudo dev/malloc-aggregation.bt -c .build/release/NIOHTTP1Server + * + * This will frequently lack symbols, so consider using the pid-based version: + * sudo dev/malloc-aggregation.bt -p 19898 + */ + +BEGIN { + printf("\n\n"); + printf("=====\n"); + printf("This will collect stack shots of allocations and print it when "); + printf("you exit bpftrace.\n"); + printf("So go ahead, run your tests and then press Ctrl+C in this window "); + printf("to see the aggregated result\n"); + printf("=====\n"); +} + +uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:aligned_alloc, +uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:calloc, +uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:malloc, +uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:posix_memalign, +uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:realloc, +uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:reallocf, +uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:valloc, +uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:malloc_zone_calloc, +uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:malloc_zone_malloc, +uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:malloc_zone_memalign, +uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:malloc_zone_realloc, +uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:malloc_zone_valloc { + @malloc_calls[ustack()] = count(); +} + +END { + print(@malloc_calls); +} diff --git a/docs/debugging-allocations.md b/docs/debugging-allocations.md index 2279aa4b845..e05399a9c77 100644 --- a/docs/debugging-allocations.md +++ b/docs/debugging-allocations.md @@ -145,16 +145,24 @@ As explained before, you can change your working directory in the temporary dire Now you should have a full Instruments allocations trace of the test run. To make sense of what's going on, switch the allocation lifespan from "Created & Persisted" to "Created & Destroyed". After that you should be able to see the type of the allocation and how many times it got allocated. Clicking on the little arrow next to the type name will reveal each individual allocation of this type and the responsible stack trace. -### Debugging with `malloc-aggregation.d` (macOS only) +### Debugging with `malloc-aggregation.d` or `malloc-aggreation.bt` -SwiftNIO ships a `dtrace` script called `dev/malloc-aggregation.d` which can give you an aggregation of all allocations by stack trace. This means that you will see the stack trace of each allocation that happened and how many times this particular stack trace allocated. +SwiftNIO ships a `dtrace` script called `dev/malloc-aggregation.d` which can give you an aggregation of all allocations by stack trace, and an equivalent `bpftrace` script called `dev/malloc-aggregation.bt`. This means that you will see the stack trace of each allocation that happened and how many times this particular stack trace allocated. -Here's an example of how to use this script: +Here's an example of how to use the `dtrace` script: ``` sudo ~/path/to/swift-nio/dev/malloc-aggregation.d -c .build/release/test_future_lots_of_callbacks ``` +The `bpftrace` script can be invoked very similarly: + +``` +sudo ~/path/to/swift-nio/dev/malloc-aggregation.bt -c .build/release/test_future_lots_of_callbacks +``` + +However, for `bpftrace` it tends to work better if pid-based tracing is used instead. This is because `bpftrace` has a [known limitation where symbolication fails if the process being traced has existed before `bpftrace` does](https://github.com/bpftrace/bpftrace/issues/2118#issuecomment-1008694821). This can still be resolved using tools like `llvm-symbolizer`, but it's trickier. + The output will look something like ``` @@ -178,7 +186,9 @@ repeated many times. Each block means that the specific stack trace is responsib When you are looking at allocations in the setup of the test the numbers may be split into one allocation set of 10000 and another of 1000 - for measured code vs warm up run. -The output from `malloc-aggreation.d` can also be diffed using the `stackdiff-dtrace.py` script. This can be helpful to track down where additional allocations were made. The `stdout` from `malloc-aggregation.d` for the two runs to compare should be written to files, and then passsed to `stackdiff-dtrace.py`: +The output from `malloc-aggreation.d` can also be diffed using the `stackdiff-dtrace.py` script. This can be helpful to track down where additional allocations were made. Right now this strategy doesn't work for `bpftrace`, but a simple modification to the `stackdiff-dtrace` script should enable it. + +The `stdout` from `malloc-aggregation.d` for the two runs to compare should be written to files, and then passsed to `stackdiff-dtrace.py`: ```bash ~/path/to/swift-nio/dev/stackdiff-dtrace.py stack_aggregation.old stack_aggregation.new @@ -312,9 +322,9 @@ before: 10000, after: 20000 Now we see there's another stacktrace in the `AFTER` section which has no corresponding stacktrace in `BEFORE`. From the stack we can see it's originating from `doRequests(group:number:)`. In this instance we were working on options applied in this function so it appears we have added allocations. We have also increased the number of allocations which are reported in the different numbers section where stack traces have been paired but with different numbers of allocations. -### Debugging with 'heaptrack' (Linux) +### Debugging with 'heaptrack' -Unfortunately we don't have dtrace or Instruments on Linux, but there is the [heaptrack](https://github.com/KDE/heaptrack) tool which supports diff analysis of two versions. +In some cases we don't have access to Linux kernel headers, which prevents us from using `bpftrace`, but there is the [heaptrack](https://github.com/KDE/heaptrack) tool which supports diff analysis of two versions. Tested on Ubuntu 20.04 (and likely working on other distributions) Install it with: From 9a2c970e9c8b355124b03b822f7d33a0b7abe51a Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 10 Feb 2025 14:19:54 +0000 Subject: [PATCH 2/3] Wildcard --- dev/malloc-aggregation.bt | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dev/malloc-aggregation.bt b/dev/malloc-aggregation.bt index 9404326f4d1..b6160a71992 100755 --- a/dev/malloc-aggregation.bt +++ b/dev/malloc-aggregation.bt @@ -31,18 +31,18 @@ BEGIN { printf("=====\n"); } -uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:aligned_alloc, -uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:calloc, -uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:malloc, -uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:posix_memalign, -uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:realloc, -uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:reallocf, -uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:valloc, -uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:malloc_zone_calloc, -uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:malloc_zone_malloc, -uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:malloc_zone_memalign, -uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:malloc_zone_realloc, -uprobe:/usr/lib/aarch64-linux-gnu/libc.so.6:malloc_zone_valloc { +uprobe:*:aligned_alloc, +uprobe:*:calloc, +uprobe:*:malloc, +uprobe:*:posix_memalign, +uprobe:*:realloc, +uprobe:*:reallocf, +uprobe:*:valloc, +uprobe:*:malloc_zone_calloc, +uprobe:*:malloc_zone_malloc, +uprobe:*:malloc_zone_memalign, +uprobe:*:malloc_zone_realloc, +uprobe:*:malloc_zone_valloc { @malloc_calls[ustack()] = count(); } From 17cf5e60385c85cc20579ce7c4392466ec71446c Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Fri, 11 Jul 2025 12:33:42 +0100 Subject: [PATCH 3/3] Update licenseignore --- .licenseignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.licenseignore b/.licenseignore index b4994d6ef7d..491d65893a5 100644 --- a/.licenseignore +++ b/.licenseignore @@ -45,5 +45,6 @@ dev/git.commit.template dev/lldb-smoker dev/make-single-file-spm dev/malloc-aggregation.d +dev/malloc-aggregation.bt dev/update-alloc-limits-to-last-completed-ci-build scripts/nio-diagnose