From 4a7205193ddde9059338ce13c3b3eed5a7adbcb6 Mon Sep 17 00:00:00 2001 From: Greg Eales <0x006EA1E5@gmail.com> Date: Thu, 20 Mar 2025 14:43:33 +0000 Subject: [PATCH] filter_nest: add wildcard_exclude config Signed-off-by: Greg Eales <0x006EA1E5@gmail.com> --- ...-bit_loki_out-structured_metadata_map.yaml | 20 +- .../docker-compose.yml | 18 +- .../docker-compose.yml | 3 + plugins/filter_nest/nest.c | 65 +++++- plugins/filter_nest/nest.h | 2 + tests/runtime/filter_nest.c | 219 ++++++++++++++++++ 6 files changed, 320 insertions(+), 7 deletions(-) diff --git a/docker_compose/loki-grafana-structured_metadata_map/config/fluent-bit_loki_out-structured_metadata_map.yaml b/docker_compose/loki-grafana-structured_metadata_map/config/fluent-bit_loki_out-structured_metadata_map.yaml index e5bac7e65b1..3a6a0951a59 100644 --- a/docker_compose/loki-grafana-structured_metadata_map/config/fluent-bit_loki_out-structured_metadata_map.yaml +++ b/docker_compose/loki-grafana-structured_metadata_map/config/fluent-bit_loki_out-structured_metadata_map.yaml @@ -1,5 +1,5 @@ service: - log_level: debug + log_level: info pipeline: inputs: @@ -20,9 +20,23 @@ pipeline: "sub_key_1": "hello, world!", "sub_key_2": "goodbye, world!" } - } + }, + "an_unknown_key": "hello, world!", + "another_unknown_key": "goodbye, world!" } + filters: + - name: nest + match: logs + nest_under: nested_for_structured_metadata + wildcard: '*' + wildcard_exclude: + - message + - logger + - hostname + - level + - 'my_map_of_*' + outputs: - name: loki match: logs @@ -31,6 +45,6 @@ pipeline: label_keys: $level,$logger labels: service_name=test structured_metadata: $hostname - structured_metadata_map_keys: $my_map_of_attributes_1,$my_map_of_maps_1['root_key'] + structured_metadata_map_keys: $my_map_of_attributes_1,$my_map_of_maps_1['root_key'],$nested_for_structured_metadata line_format: key_value drop_single_key: on diff --git a/docker_compose/loki-grafana-structured_metadata_map/docker-compose.yml b/docker_compose/loki-grafana-structured_metadata_map/docker-compose.yml index e2a06d8b11e..618a4eaa713 100644 --- a/docker_compose/loki-grafana-structured_metadata_map/docker-compose.yml +++ b/docker_compose/loki-grafana-structured_metadata_map/docker-compose.yml @@ -6,7 +6,8 @@ services: # context: ../../ # dockerfile: dockerfiles/Dockerfile depends_on: - - loki + loki: + condition: service_healthy container_name: fluentbit command: /fluent-bit/bin/fluent-bit -c /etc/fluent-bit_loki_out-structured_metadata_map.yaml ports: @@ -17,7 +18,8 @@ services: - ./config/fluent-bit_loki_out-structured_metadata_map.yaml:/etc/fluent-bit_loki_out-structured_metadata_map.yaml grafana: - image: grafana/grafana:11.4.0 + image: grafana/grafana:11.5.2 + attach: false depends_on: - loki - fluentbit @@ -28,11 +30,21 @@ services: networks: - loki-network environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_SECURITY_ADMIN=admin - GF_SECURITY_ADMIN_PASSWORD=admin loki: - image: grafana/loki:2.9.2 + image: grafana/loki:3.4.2 + attach: false command: -config.file=/etc/loki/loki-config.yaml + healthcheck: + test: [ "CMD", "wget", "-q", "-O", "/dev/null", "http://localhost:3100/ready" ] + interval: 5s + retries: 10 + start_period: 10s + timeout: 10s networks: - loki-network ports: diff --git a/docker_compose/node-exporter-dashboard/docker-compose.yml b/docker_compose/node-exporter-dashboard/docker-compose.yml index 8170ee7ebf4..84a67f52167 100644 --- a/docker_compose/node-exporter-dashboard/docker-compose.yml +++ b/docker_compose/node-exporter-dashboard/docker-compose.yml @@ -24,6 +24,9 @@ services: networks: - exporter-network environment: + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_SECURITY_ADMIN=admin - GF_SECURITY_ADMIN_PASSWORD=admin prometheus: diff --git a/plugins/filter_nest/nest.c b/plugins/filter_nest/nest.c index 2ace2137e81..042f0ae3670 100644 --- a/plugins/filter_nest/nest.c +++ b/plugins/filter_nest/nest.c @@ -41,6 +41,7 @@ static void teardown(struct filter_nest_ctx *ctx) struct mk_list *head; struct filter_nest_wildcard *wildcard; + struct filter_nest_wildcard *wildcard_excludes; flb_free(ctx->prefix); flb_free(ctx->key); @@ -51,6 +52,12 @@ static void teardown(struct filter_nest_ctx *ctx) mk_list_del(&wildcard->_head); flb_free(wildcard); } + mk_list_foreach_safe(head, tmp, &ctx->wildcard_excludes) { + wildcard_excludes = mk_list_entry(head, struct filter_nest_wildcard, _head); + flb_free(wildcard_excludes->key); + mk_list_del(&wildcard_excludes->_head); + flb_free(wildcard_excludes); + } } @@ -62,6 +69,7 @@ static int configure(struct filter_nest_ctx *ctx, struct mk_list *head; struct flb_kv *kv; struct filter_nest_wildcard *wildcard; + struct filter_nest_wildcard *wildcard_exclude; char *operation_nest = "nest"; char *operation_lift = "lift"; @@ -123,6 +131,35 @@ static int configure(struct filter_nest_ctx *ctx, mk_list_add(&wildcard->_head, &ctx->wildcards); ctx->wildcards_cnt++; + } + else if (strcasecmp(kv->key, "wildcard_exclude") == 0) { + wildcard_exclude = flb_malloc(sizeof(struct filter_nest_wildcard)); + if (!wildcard_exclude) { + flb_plg_error(ctx->ins, "Unable to allocate memory for " + "wildcard_exclude"); + flb_free(wildcard_exclude); + return -1; + } + + wildcard_exclude->key = flb_strndup(kv->val, flb_sds_len(kv->val)); + if (wildcard_exclude->key == NULL) { + flb_errno(); + flb_free(wildcard_exclude); + return -1; + } + wildcard_exclude->key_len = flb_sds_len(kv->val); + + if (wildcard_exclude->key[wildcard_exclude->key_len - 1] == '*') { + wildcard_exclude->key_is_dynamic = true; + wildcard_exclude->key_len--; + } + else { + wildcard_exclude->key_is_dynamic = false; + } + + mk_list_add(&wildcard_exclude->_head, &ctx->wildcard_excludes); + ctx->wildcard_excludes_cnt++; + } else if (strcasecmp(kv->key, "nest_under") == 0) { ctx->key = flb_strdup(kv->val); @@ -307,6 +344,7 @@ static inline bool is_kv_to_nest(msgpack_object_kv * kv, struct mk_list *tmp; struct mk_list *head; struct filter_nest_wildcard *wildcard; + struct filter_nest_wildcard *wildcard_exclude; if (obj->type == MSGPACK_OBJECT_BIN) { key = obj->via.bin.ptr; @@ -321,6 +359,24 @@ static inline bool is_kv_to_nest(msgpack_object_kv * kv, return false; } + mk_list_foreach_safe(head, tmp, &ctx->wildcard_excludes) { + wildcard_exclude = mk_list_entry(head, struct filter_nest_wildcard, _head); + + if (wildcard_exclude->key_is_dynamic) { + /* This will negatively match "ABC123" with prefix "ABC*" */ + if (strncmp(key, wildcard_exclude->key, wildcard_exclude->key_len) == 0) { + return false; + } + } + else { + /* This will negatively match "ABC" with prefix "ABC" */ + if ((wildcard_exclude->key_len == klen) && + (strncmp(key, wildcard_exclude->key, klen) == 0) + ) { + return false; + } + } + } mk_list_foreach_safe(head, tmp, &ctx->wildcards) { wildcard = mk_list_entry(head, struct filter_nest_wildcard, _head); @@ -615,9 +671,11 @@ static int cb_nest_init(struct flb_filter_instance *f_ins, flb_errno(); return -1; } - mk_list_init(&ctx->wildcards); ctx->ins = f_ins; + mk_list_init(&ctx->wildcards); ctx->wildcards_cnt = 0; + mk_list_init(&ctx->wildcard_excludes); + ctx->wildcard_excludes_cnt = 0; if (configure(ctx, f_ins, config) < 0) { flb_free(ctx); @@ -737,6 +795,11 @@ static struct flb_config_map config_map[] = { FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, "Nest records which field matches the wildcard" }, + { + FLB_CONFIG_MAP_STR, "Wildcard_exclude", NULL, + FLB_CONFIG_MAP_MULT, FLB_FALSE, 0, + "Nest records which field doesn't matches the wildcard" + }, { FLB_CONFIG_MAP_STR, "Nest_under", NULL, 0, FLB_FALSE, 0, diff --git a/plugins/filter_nest/nest.h b/plugins/filter_nest/nest.h index 678c79a836b..49aae8ee3e8 100644 --- a/plugins/filter_nest/nest.h +++ b/plugins/filter_nest/nest.h @@ -38,6 +38,8 @@ struct filter_nest_ctx // nest struct mk_list wildcards; int wildcards_cnt; + struct mk_list wildcard_excludes; + int wildcard_excludes_cnt; bool remove_prefix; // lift bool add_prefix; diff --git a/tests/runtime/filter_nest.c b/tests/runtime/filter_nest.c index f9b43814f0f..2ce1f0a0663 100644 --- a/tests/runtime/filter_nest.c +++ b/tests/runtime/filter_nest.c @@ -15,12 +15,18 @@ void flb_test_filter_nest_single(void); void flb_test_filter_nest_multi_nest(void); void flb_test_filter_nest_multi_lift(void); void flb_test_filter_nest_add_prefix(void); +void flb_test_filter_nest_include_all(void); +void flb_test_filter_nest_exclude_static(void); +void flb_test_filter_nest_exclude_wildcard(void); /* Test list */ TEST_LIST = { {"single", flb_test_filter_nest_single }, {"multiple events are not dropped(nest)", flb_test_filter_nest_multi_nest}, {"multiple events are not dropped(lift)", flb_test_filter_nest_multi_lift}, {"add_prefix", flb_test_filter_nest_add_prefix}, + {"include_all", flb_test_filter_nest_include_all}, + {"exclude_static", flb_test_filter_nest_exclude_static}, + {"exclude_wildcard", flb_test_filter_nest_exclude_wildcard}, {NULL, NULL} }; @@ -372,3 +378,216 @@ void flb_test_filter_nest_add_prefix(void) flb_stop(ctx); flb_destroy(ctx); } + +void flb_test_filter_nest_include_all(void) +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + + ctx = flb_create(); + flb_service_set(ctx, + "Flush", "1", + "Grace", "1", + "Log_Level", "debug", + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "nest", NULL); + TEST_CHECK(filter_ffd >= 0); + + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Operation", "nest", + "Nest_under", "nested_key", + "Wildcard", "*", + NULL); + + TEST_CHECK(ret == 0); + + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *) &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + p = "[1448403340, {\"to_nest\":\"This is some data to nest\", \"extra\":\"Some more data to nest\", \"EXCLUDED_extra_1\":\"Some more data to be excluded\", \"EXCLUDED_extra_2\":\"Some more data to be excluded\", \"EXCLUDED_extra_3\":\"Some more data to be excluded\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + flb_time_msleep(1500); /* waiting flush */ + output = get_output(); + + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + + if (output != NULL) { + expected = "{\"nested_key\":{\"to_nest\":\"This is some data to nest\",\"extra\":\"Some more data to nest\",\"EXCLUDED_extra_1\":\"Some more data to be excluded\",\"EXCLUDED_extra_2\":\"Some more data to be excluded\",\"EXCLUDED_extra_3\":\"Some more data to be excluded\"}}]"; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + free(output); + } + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_nest_exclude_static(void) +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + + ctx = flb_create(); + flb_service_set(ctx, + "Flush", "1", + "Grace", "1", + "Log_Level", "debug", + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "nest", NULL); + TEST_CHECK(filter_ffd >= 0); + + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Operation", "nest", + "Nest_under", "nested_key", + "Wildcard", "*", + "Wildcard_exclude", "EXCLUDED_extra_1", + "Wildcard_exclude", "EXCLUDED_extra_2", + NULL); + + TEST_CHECK(ret == 0); + + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *) &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + p = "[1448403340, {\"to_nest\":\"This is some data to nest\", \"extra\":\"Some more data to nest\", \"EXCLUDED_extra_1\":\"Some more data to be excluded\", \"EXCLUDED_extra_2\":\"Some more data to be excluded\", \"EXCLUDED_NOT\":\"Some more data to be excluded\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + flb_time_msleep(1500); /* waiting flush */ + output = get_output(); + + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + + if (output != NULL) { + expected = "\"nested_key\":{\"to_nest\":\"This is some data to nest\",\"extra\":\"Some more data to nest\",\"EXCLUDED_NOT\":\"Some more data to be excluded\"}}"; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + free(output); + } + flb_stop(ctx); + flb_destroy(ctx); +} + +void flb_test_filter_nest_exclude_wildcard(void) +{ + int ret; + int bytes; + char *p, *output, *expected; + flb_ctx_t *ctx; + int in_ffd; + int out_ffd; + int filter_ffd; + struct flb_lib_out_cb cb_data; + + ctx = flb_create(); + flb_service_set(ctx, + "Flush", "1", + "Grace", "1", + "Log_Level", "debug", + NULL); + + /* Input */ + in_ffd = flb_input(ctx, (char *) "lib", NULL); + TEST_CHECK(in_ffd >= 0); + flb_input_set(ctx, in_ffd, "tag", "test", NULL); + + /* Prepare output callback context*/ + cb_data.cb = callback_test; + cb_data.data = NULL; + + /* Filter */ + filter_ffd = flb_filter(ctx, (char *) "nest", NULL); + TEST_CHECK(filter_ffd >= 0); + + ret = flb_filter_set(ctx, filter_ffd, + "Match", "*", + "Operation", "nest", + "Nest_under", "nested_key", + "Wildcard", "*", + "Wildcard_exclude", "EXCLUDED_*", + NULL); + + TEST_CHECK(ret == 0); + + + /* Output */ + out_ffd = flb_output(ctx, (char *) "lib", (void *) &cb_data); + TEST_CHECK(out_ffd >= 0); + flb_output_set(ctx, out_ffd, + "match", "test", + "format", "json", + NULL); + + ret = flb_start(ctx); + TEST_CHECK(ret == 0); + + p = "[1448403340, {\"to_nest\":\"This is some data to nest\", \"extra\":\"Some more data to nest\", \"EXCLUDED_extra_1\":\"Some more data to be excluded\", \"EXCLUDED_extra_2\":\"Some more data to be excluded\", \"EXCLUDED_extra_3\":\"Some more data to be excluded\"}]"; + bytes = flb_lib_push(ctx, in_ffd, p, strlen(p)); + TEST_CHECK(bytes == strlen(p)); + + flb_time_msleep(1500); /* waiting flush */ + output = get_output(); + + TEST_CHECK_(output != NULL, "Expected output to not be NULL"); + + if (output != NULL) { + expected = "\"nested_key\":{\"to_nest\":\"This is some data to nest\",\"extra\":\"Some more data to nest\"}}]"; + TEST_CHECK_(strstr(output, expected) != NULL, "Expected output to contain '%s', got '%s'", expected, output); + free(output); + } + flb_stop(ctx); + flb_destroy(ctx); +} \ No newline at end of file