Skip to content

filter_nest: add wildcard_exclude config #10112

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
0x006EA1E5 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

filter_nest: add wildcard_exclude config #10112

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 17 additions & 3 deletions ...i-grafana-structured_metadata_map/config/fluent-bit_loki_out-structured_metadata_map.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
service:
log_level: debug
log_level: info

pipeline:
inputs:
Expand All @@ -20,9 +20,23 @@ pipeline:
"sub_key_1": "hello, world!",
"sub_key_2": "goodbye, world!"
}
}
},
"an_unknown_key": "hello, world!",
"another_unknown_key": "goodbye, world!"
}

filters:
- name: nest
match: logs
nest_under: nested_for_structured_metadata
wildcard: '*'
wildcard_exclude:
- message
- logger
- hostname
- level
- 'my_map_of_*'

outputs:
- name: loki
match: logs
Expand All @@ -31,6 +45,6 @@ pipeline:
label_keys: $level,$logger
labels: service_name=test
structured_metadata: $hostname
structured_metadata_map_keys: $my_map_of_attributes_1,$my_map_of_maps_1['root_key']
structured_metadata_map_keys: $my_map_of_attributes_1,$my_map_of_maps_1['root_key'],$nested_for_structured_metadata
line_format: key_value
drop_single_key: on
18 changes: 15 additions & 3 deletions docker_compose/loki-grafana-structured_metadata_map/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ services:
# context: ../../
# dockerfile: dockerfiles/Dockerfile
depends_on:
- loki
loki:
condition: service_healthy
container_name: fluentbit
command: /fluent-bit/bin/fluent-bit -c /etc/fluent-bit_loki_out-structured_metadata_map.yaml
ports:
Expand All @@ -17,7 +18,8 @@ services:
- ./config/fluent-bit_loki_out-structured_metadata_map.yaml:/etc/fluent-bit_loki_out-structured_metadata_map.yaml

grafana:
image: grafana/grafana:11.4.0
image: grafana/grafana:11.5.2
attach: false
depends_on:
- loki
- fluentbit
Expand All @@ -28,11 +30,21 @@ services:
networks:
- loki-network
environment:
- GF_AUTH_ANONYMOUS_ENABLED=true
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
- GF_SECURITY_ADMIN=admin
- GF_SECURITY_ADMIN_PASSWORD=admin

loki:
image: grafana/loki:2.9.2
image: grafana/loki:3.4.2
attach: false
command: -config.file=/etc/loki/loki-config.yaml
healthcheck:
test: [ "CMD", "wget", "-q", "-O", "/dev/null", "http://localhost:3100/ready" ]
interval: 5s
retries: 10
start_period: 10s
timeout: 10s
networks:
- loki-network
ports:
Expand Down
3 changes: 3 additions & 0 deletions docker_compose/node-exporter-dashboard/docker-compose.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@ services:
networks:
- exporter-network
environment:
- GF_AUTH_ANONYMOUS_ENABLED=true
- GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
- GF_SECURITY_ADMIN=admin
- GF_SECURITY_ADMIN_PASSWORD=admin

prometheus:
Expand Down
65 changes: 64 additions & 1 deletion plugins/filter_nest/nest.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ static void teardown(struct filter_nest_ctx *ctx)
struct mk_list *head;

struct filter_nest_wildcard *wildcard;
struct filter_nest_wildcard *wildcard_excludes;

flb_free(ctx->prefix);
flb_free(ctx->key);
Expand All @@ -51,6 +52,12 @@ static void teardown(struct filter_nest_ctx *ctx)
mk_list_del(&wildcard->_head);
flb_free(wildcard);
}
mk_list_foreach_safe(head, tmp, &ctx->wildcard_excludes) {
wildcard_excludes = mk_list_entry(head, struct filter_nest_wildcard, _head);
flb_free(wildcard_excludes->key);
mk_list_del(&wildcard_excludes->_head);
flb_free(wildcard_excludes);
}

}

Expand All @@ -62,6 +69,7 @@ static int configure(struct filter_nest_ctx *ctx,
struct mk_list *head;
struct flb_kv *kv;
struct filter_nest_wildcard *wildcard;
struct filter_nest_wildcard *wildcard_exclude;

char *operation_nest = "nest";
char *operation_lift = "lift";
Expand Down Expand Up @@ -123,6 +131,35 @@ static int configure(struct filter_nest_ctx *ctx,
mk_list_add(&wildcard->_head, &ctx->wildcards);
ctx->wildcards_cnt++;

}
else if (strcasecmp(kv->key, "wildcard_exclude") == 0) {
wildcard_exclude = flb_malloc(sizeof(struct filter_nest_wildcard));
if (!wildcard_exclude) {
flb_plg_error(ctx->ins, "Unable to allocate memory for "
"wildcard_exclude");
flb_free(wildcard_exclude);
return -1;
}

wildcard_exclude->key = flb_strndup(kv->val, flb_sds_len(kv->val));
if (wildcard_exclude->key == NULL) {
flb_errno();
flb_free(wildcard_exclude);
return -1;
}
wildcard_exclude->key_len = flb_sds_len(kv->val);

if (wildcard_exclude->key[wildcard_exclude->key_len - 1] == '*') {
wildcard_exclude->key_is_dynamic = true;
wildcard_exclude->key_len--;
}
else {
wildcard_exclude->key_is_dynamic = false;
}

mk_list_add(&wildcard_exclude->_head, &ctx->wildcard_excludes);
ctx->wildcard_excludes_cnt++;

}
else if (strcasecmp(kv->key, "nest_under") == 0) {
ctx->key = flb_strdup(kv->val);
Expand Down Expand Up @@ -307,6 +344,7 @@ static inline bool is_kv_to_nest(msgpack_object_kv * kv,
struct mk_list *tmp;
struct mk_list *head;
struct filter_nest_wildcard *wildcard;
struct filter_nest_wildcard *wildcard_exclude;

if (obj->type == MSGPACK_OBJECT_BIN) {
key = obj->via.bin.ptr;
Expand All @@ -321,6 +359,24 @@ static inline bool is_kv_to_nest(msgpack_object_kv * kv,
return false;
}

mk_list_foreach_safe(head, tmp, &ctx->wildcard_excludes) {
wildcard_exclude = mk_list_entry(head, struct filter_nest_wildcard, _head);

if (wildcard_exclude->key_is_dynamic) {
/* This will negatively match "ABC123" with prefix "ABC*" */
if (strncmp(key, wildcard_exclude->key, wildcard_exclude->key_len) == 0) {
Copy link
Copy Markdown
Member

@edsiper edsiper Mar 27, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

key is not a null terminated string and this comparison can potentially fail. Since klen is set, I recommend comparing first klen == wildcard_exclude->key_len and if it passes then do the strncmp()

Copy link
Copy Markdown
Contributor Author

@0x006EA1E5 0x006EA1E5 Mar 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FYI, I copy and pasted this bloc of code from wildcards below, and adapted to wildcards_excludes.

As I read it, this part is for "dynamic" matches (where there is a wildcard), so we don't expect the lengths to match. For "non-dynamic" (which follows), it does do a length check already.

However, we could do a check here, something like klen >= wildcard_exclude->key_len, right?

This is to check that ABC123 is a viable match for wildcard ABC*, so we want to check that ABC123 is at least three chars long...

I will add some more unit tests to verify all this 🤞

Copy link
Copy Markdown
Contributor Author

@0x006EA1E5 0x006EA1E5 Mar 28, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added the klen >= wildcard_exclude->key_len check, and some tests, and it all looks good

return false;
}
}
else {
/* This will negatively match "ABC" with prefix "ABC" */
if ((wildcard_exclude->key_len == klen) &&
(strncmp(key, wildcard_exclude->key, klen) == 0)
) {
return false;
}
}
}
mk_list_foreach_safe(head, tmp, &ctx->wildcards) {
wildcard = mk_list_entry(head, struct filter_nest_wildcard, _head);

Expand Down Expand Up @@ -615,9 +671,11 @@ static int cb_nest_init(struct flb_filter_instance *f_ins,
flb_errno();
return -1;
}
mk_list_init(&ctx->wildcards);
ctx->ins = f_ins;
mk_list_init(&ctx->wildcards);
ctx->wildcards_cnt = 0;
mk_list_init(&ctx->wildcard_excludes);
ctx->wildcard_excludes_cnt = 0;

if (configure(ctx, f_ins, config) < 0) {
flb_free(ctx);
Expand Down Expand Up @@ -737,6 +795,11 @@ static struct flb_config_map config_map[] = {
FLB_CONFIG_MAP_MULT, FLB_FALSE, 0,
"Nest records which field matches the wildcard"
},
{
FLB_CONFIG_MAP_STR, "Wildcard_exclude", NULL,
FLB_CONFIG_MAP_MULT, FLB_FALSE, 0,
"Nest records which field doesn't matches the wildcard"
},
{
FLB_CONFIG_MAP_STR, "Nest_under", NULL,
0, FLB_FALSE, 0,
Expand Down
2 changes: 2 additions & 0 deletions plugins/filter_nest/nest.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ struct filter_nest_ctx
// nest
struct mk_list wildcards;
int wildcards_cnt;
struct mk_list wildcard_excludes;
int wildcard_excludes_cnt;
bool remove_prefix;
// lift
bool add_prefix;
Expand Down
Loading
Loading