Skip to content

Add distributed tracing for k6-backed checks #923

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Add distributed tracing for k6-backed checks #923

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
c14463d
tracing: add TracerProvider and initiate traces for check runs
nadiamoe Oct 2, 2024
e8f09b8
main: initialize tracerprovider and hand it down to handlers
nadiamoe Oct 2, 2024
717bce6
k6runner: create spans for k6-backed check runs
nadiamoe Oct 2, 2024
9ff0ca4
k6runner/http: create spans in k6 http runner
nadiamoe Oct 3, 2024
dc8d27d
Pass argument on the command line to specify OTLP endpoint
mem Oct 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 71 additions & 6 deletions cmd/synthetic-monitoring-agent/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ import (
"github.com/jpillora/backoff"
"github.com/prometheus/client_golang/prometheus"
"github.com/rs/zerolog"
otelGlobal "go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.26.0"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/noop"
"golang.org/x/sync/errgroup"
"google.golang.org/grpc/grpclog"

Expand Down Expand Up @@ -73,6 +80,7 @@ func run(args []string, stdout io.Writer) error {
AutoMemLimit bool
MemLimitRatio float64
DisableK6 bool
OTLPEndpoint string
}{
GrpcApiServerAddr: "localhost:4031",
HttpListenAddr: "localhost:4050",
Expand Down Expand Up @@ -104,6 +112,7 @@ func run(args []string, stdout io.Writer) error {
flags.BoolVar(&config.DisableK6, "disable-k6", config.DisableK6, "disables running k6 checks on this probe")
flags.Float64Var(&config.MemLimitRatio, "memlimit-ratio", config.MemLimitRatio, "fraction of available memory to use")
flags.Var(&features, "features", "optional feature flags")
flags.StringVar(&config.OTLPEndpoint, "otlp-endpoint", config.OTLPEndpoint, "OTLP endpoint for OTEL exporter")

if err := flags.Parse(args[1:]); err != nil {
return err
Expand Down Expand Up @@ -152,13 +161,7 @@ func run(args []string, stdout io.Writer) error {
return fmt.Errorf("invalid API token")
}

baseCtx, cancel := context.WithCancel(context.Background())
defer cancel()

g, ctx := errgroup.WithContext(baseCtx)

zerolog.TimeFieldFormat = zerolog.TimeFormatUnixMs

zl := zerolog.New(stdout).With().Timestamp().Str("program", filepath.Base(args[0])).Logger()

switch {
Expand All @@ -176,6 +179,11 @@ func run(args []string, stdout io.Writer) error {
zerolog.SetGlobalLevel(zerolog.ErrorLevel)
}

baseCtx, cancel := context.WithCancel(context.Background())
defer cancel()

g, ctx := errgroup.WithContext(baseCtx)

g.Go(func() error {
return signalHandler(ctx, zl.With().Str("subsystem", "signal handler").Logger())
})
Expand Down Expand Up @@ -208,6 +216,15 @@ func run(args []string, stdout io.Writer) error {
zl.Info().Msg("disabling k6 checks")
}

tracerProvider, err := newTracerProvider(ctx, config.OTLPEndpoint, flags.Name())
if err != nil {
zl.Warn().Err(err).Msg("could not configure tracer provider, disabling tracing functionality")
tracerProvider = noop.NewTracerProvider()
}

// Set the global tracer provider to no-op, in case some library tries to obtain one that way.
otelGlobal.SetTracerProvider(noop.NewTracerProvider())

promRegisterer := prometheus.NewRegistry()

if err := registerMetrics(promRegisterer); err != nil {
Expand Down Expand Up @@ -299,6 +316,7 @@ func run(args []string, stdout io.Writer) error {
checksUpdater, err := checks.NewUpdater(checks.UpdaterOptions{
Conn: conn,
Logger: zl.With().Str("subsystem", "updater").Logger(),
TracerProvider: tracerProvider,
Backoff: newConnectionBackoff(),
Publisher: publisher,
TenantCh: tenantCh,
Expand All @@ -321,6 +339,7 @@ func run(args []string, stdout io.Writer) error {
adhocHandler, err := adhoc.NewHandler(adhoc.HandlerOpts{
Conn: conn,
Logger: zl.With().Str("subsystem", "adhoc").Logger(),
TracerProvider: tracerProvider,
Backoff: newConnectionBackoff(),
Publisher: publisher,
TenantCh: tenantCh,
Expand Down Expand Up @@ -439,3 +458,49 @@ func setupGoMemLimit(ratio float64) error {

return nil
}

// newTracerProvider creates a TracerProvider configured with the relevant
// resource attributes, configured to export traces over OTLP+HTTP to the
// specified endpoint. The provided name will be used to identify the service
// in traces.
//
// The context is needed so that the exporter can be stopped when the context
// is cancelled, so it's probably a good idea to use a context that is
// cancelled when the program is shutting down.
//
// If no endpoint is provided, a no-op tracer provider is returned and no error
// is reported.
//
// After this function returns successfully, the tracer exporter would have
// been started.
func newTracerProvider(ctx context.Context, endpoint, name string) (trace.TracerProvider, error) {
if endpoint == "" {
return noop.NewTracerProvider(), nil
}

if _, err := url.Parse(endpoint); err != nil {
return nil, fmt.Errorf("invalid OTLP endpoint URL: %w", err)
}

res, err := resource.Merge(
resource.Default(),
resource.NewWithAttributes(
semconv.SchemaURL,
semconv.ServiceName(name),
semconv.ServiceVersion(version.Short()),
),
)
if err != nil {
return nil, fmt.Errorf("creating otel resources: %w", err)
}

te, err := otlptracehttp.New(ctx, otlptracehttp.WithEndpointURL(endpoint))
if err != nil {
return nil, fmt.Errorf("starting otelhttp trace exporter: %w", err)
}

return sdktrace.NewTracerProvider(
sdktrace.WithBatcher(te),
sdktrace.WithResource(res),
), nil
}
17 changes: 15 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,12 @@ require (
github.com/mccutchen/go-httpbin/v2 v2.15.0
github.com/quasilyte/go-ruleguard/dsl v0.3.22
github.com/spf13/afero v1.11.0
golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0
go.opentelemetry.io/otel v1.31.0
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0
go.opentelemetry.io/otel/sdk v1.30.0
go.opentelemetry.io/otel/trace v1.31.0
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
gopkg.in/yaml.v3 v3.0.1
kernel.org/pub/linux/libs/security/libcap/cap v1.2.70
)
Expand All @@ -47,16 +52,20 @@ require (
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/buger/goterm v1.0.4 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cilium/ebpf v0.11.0 // indirect
github.com/containerd/cgroups/v3 v3.0.3 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/dennwc/varint v1.0.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/godbus/dbus/v5 v5.0.4 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 // indirect
github.com/klauspost/compress v1.17.9 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
Expand All @@ -67,13 +76,17 @@ require (
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/procfs v0.15.1 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 // indirect
go.opentelemetry.io/otel/metric v1.31.0 // indirect
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
go.uber.org/atomic v1.11.0 // indirect
golang.org/x/mod v0.21.0 // indirect
golang.org/x/oauth2 v0.23.0 // indirect
golang.org/x/sys v0.26.0 // indirect
golang.org/x/text v0.19.0 // indirect
golang.org/x/tools v0.26.0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect
google.golang.org/protobuf v1.34.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
kernel.org/pub/linux/libs/security/libcap/psx v1.2.70 // indirect
Expand Down
37 changes: 31 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
github.com/buger/goterm v0.0.0-20181115115552-c206103e1f37/go.mod h1:u9UyCz2eTrSGy6fbupqJ54eY5c4IC8gREQ1053dK12U=
github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY=
github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cilium/ebpf v0.11.0 h1:V8gS/bTCCjX9uUnkUFUpPsksM8n1lXBAvHcpiFk1X2Y=
Expand Down Expand Up @@ -48,8 +50,11 @@ github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU=
github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0=
github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/go-ping/ping v1.1.0 h1:3MCGhVX4fyEUuhsfwPrsEdQw6xspHkv5zHsiSoDFZYw=
github.com/go-ping/ping v1.1.0/go.mod h1:xIFjORFzTxqIV/tDVGO4eDy/bLuSyawEeojSm3GfRGk=
github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA=
Expand Down Expand Up @@ -79,6 +84,8 @@ github.com/grafana/mtr v0.1.1-0.20221107202107-a9806fdda166 h1:COSDtVDArtLKK9p+m
github.com/grafana/mtr v0.1.1-0.20221107202107-a9806fdda166/go.mod h1:qDO1rp1hUZzunyD0f38VNbY0j6k+ZFRNZkHl3jbn/MU=
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc h1:GN2Lv3MGO7AS6PrRoT6yV5+wkrOpcszoIsO4+4ds248=
github.com/grafana/regexp v0.0.0-20240518133315-a468a5bfb3bc/go.mod h1:+JKpmjMGhpgPL+rXZ5nsZieVzvarn86asRlBg4uNGnk=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0 h1:asbCHRVmodnJTuQ3qamDwqVOIjwqUPTYmYuemVOx+Ys=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.22.0/go.mod h1:ggCgvZ2r7uOoQjOyu2Y1NhHmEPPzzuhWgcza5M1Ji1I=
github.com/hokaccha/go-prettyjson v0.0.0-20180920040306-f579f869bbfe/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
Expand Down Expand Up @@ -142,8 +149,8 @@ github.com/prometheus/prometheus v0.54.1 h1:vKuwQNjnYN2/mDoWfHXDhAsz/68q/dQDb+Yb
github.com/prometheus/prometheus v0.54.1/go.mod h1:xlLByHhk2g3ycakQGrMaU8K7OySZx98BzeCR99991NY=
github.com/quasilyte/go-ruleguard/dsl v0.3.22 h1:wd8zkOhSNr+I+8Qeciml08ivDt1pSXe60+5DqOpCjPE=
github.com/quasilyte/go-ruleguard/dsl v0.3.22/go.mod h1:KeCP03KrjuSO0H1kTuZQCWlQPulDV6YMIXmpQss17rU=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
Expand All @@ -165,6 +172,22 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg=
go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY=
go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0 h1:lsInsfvhVIfOI6qHVyysXMNDnjO9Npvl7tlDPJFBVd4=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.30.0/go.mod h1:KQsVNh4OjgjTG0G6EiNi1jVpnaeeKsKMRwbLN+f1+8M=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0 h1:umZgi92IyxfXd/l4kaDhnKgY8rnN/cZcF1LKc6I8OQ8=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.30.0/go.mod h1:4lVs6obhSVRb1EW5FhOuBTyiQhtRtAnnva9vD3yRfq8=
go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE=
go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY=
go.opentelemetry.io/otel/sdk v1.30.0 h1:cHdik6irO49R5IysVhdn8oaiR9m8XluDaJAs4DfOrYE=
go.opentelemetry.io/otel/sdk v1.30.0/go.mod h1:p14X4Ok8S+sygzblytT1nqG98QG2KYKv++HE0LY/mhg=
go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys=
go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A=
go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
Expand All @@ -174,8 +197,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6 h1:1wqE9dj9NpSm04INVsJhhEUzhuDVjbcyKH91sVyPATw=
golang.org/x/exp v0.0.0-20241004190924-225e2abe05e6/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8=
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk=
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0=
Expand Down Expand Up @@ -226,8 +249,10 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 h1:hjSy6tcFQZ171igDaN5QHOw2n6vx40juYbC/x67CEhc=
google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:qpvKtACPCQhAdu3PyQgV4l3LMXZEtft7y8QcarRsp9I=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 h1:pPJltXNxVzT4pK9yD8vR9X75DaWYYmLGMsEvBfFQZzQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU=
google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E=
google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
Expand Down
18 changes: 18 additions & 0 deletions internal/adhoc/adhoc.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/prometheus/prompb"
"github.com/rs/zerolog"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/connectivity"
Expand All @@ -32,6 +34,7 @@ import (
type Handler struct {
api apiInfo
logger zerolog.Logger
tracerProvider trace.TracerProvider
features feature.Collection
backoff Backoffer
probe *sm.Probe
Expand Down Expand Up @@ -101,6 +104,7 @@ func (b constantBackoff) Duration() time.Duration { return time.Duration(b) }
type HandlerOpts struct {
Conn ClientConn
Logger zerolog.Logger
TracerProvider trace.TracerProvider
Backoff Backoffer
Publisher pusher.Publisher
TenantCh chan<- sm.Tenant
Expand Down Expand Up @@ -133,6 +137,7 @@ func NewHandler(opts HandlerOpts) (*Handler, error) {

h := &Handler{
logger: opts.Logger,
tracerProvider: opts.TracerProvider,
features: opts.Features,
backoff: opts.Backoff,
publisher: opts.Publisher,
Expand Down Expand Up @@ -349,6 +354,15 @@ func (h *Handler) processAdHocChecks(ctx context.Context, client sm.AdHocChecks_
}

func (h *Handler) handleAdHocCheck(ctx context.Context, ahReq *sm.AdHocRequest) error {
ctx, adHocSpan := h.tracerProvider.Tracer("").Start(ctx, "adHoc check")
defer adHocSpan.End()
adHocSpan.SetAttributes(
attribute.String("type", ahReq.AdHocCheck.Type().String()),
attribute.Int64("tenantId", ahReq.AdHocCheck.TenantId),
attribute.String("id", ahReq.AdHocCheck.Id),
)
adHocSpan.SetAttributes(attribute.String("type", ahReq.AdHocCheck.Type().String()))

h.logger.Debug().Interface("request", ahReq).Msg("got ad-hoc check request")

h.metrics.opsCounter.WithLabelValues(ahReq.AdHocCheck.Type().String()).Inc()
Expand Down Expand Up @@ -479,10 +493,14 @@ func (r *runner) Run(ctx context.Context, tenantId model.GlobalID, publisher pus

start := time.Now()

ctx, runSpan := trace.SpanFromContext(ctx).TracerProvider().Tracer("").Start(ctx, "adHoc run")
defer runSpan.End()

rCtx, cancel := context.WithTimeout(ctx, r.timeout)
defer cancel()

success, duration := r.prober.Probe(rCtx, r.target, registry, logger)
runSpan.SetAttributes(attribute.Bool("success", success), attribute.Float64("durationSeconds", duration))

if success {
successGauge.Set(1)
Expand Down
7 changes: 5 additions & 2 deletions internal/adhoc/adhoc_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/rs/zerolog"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/trace/noop"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/connectivity"
Expand All @@ -29,6 +30,7 @@ func TestNewHandler(t *testing.T) {
opts := HandlerOpts{
Conn: nil,
Logger: zerolog.New(io.Discard),
TracerProvider: noop.NewTracerProvider(),
Publisher: channelPublisher(make(chan pusher.Payload)),
TenantCh: make(chan sm.Tenant),
PromRegisterer: prometheus.NewPedanticRegistry(),
Expand Down Expand Up @@ -63,6 +65,7 @@ func TestHandlerRun(t *testing.T) {

opts := HandlerOpts{
Logger: logger,
TracerProvider: noop.NewTracerProvider(),
Publisher: channelPublisher(publishCh),
TenantCh: make(chan sm.Tenant),
PromRegisterer: prometheus.NewPedanticRegistry(),
Expand Down Expand Up @@ -132,8 +135,7 @@ func (testBackoff) Duration() time.Duration {

func (testBackoff) Reset() {}

type grpcTestConn struct {
}
type grpcTestConn struct{}

func (grpcTestConn) GetState() connectivity.State {
return connectivity.Ready
Expand Down Expand Up @@ -189,6 +191,7 @@ func TestHandlerRunErrors(t *testing.T) {
opts := HandlerOpts{
Conn: &grpcTestConn{},
Logger: logger,
TracerProvider: noop.NewTracerProvider(),
Publisher: channelPublisher(publishCh),
Backoff: testBackoff{},
TenantCh: make(chan sm.Tenant),
Expand Down
Loading