From 2872d81fafac341272bf8e1cadd9b503fac715d4 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 15 Jul 2025 06:24:49 +0300 Subject: [PATCH 1/6] Enabling failure store for log data streams --- .../datastreams/LogsDataStreamIT.java | 68 ++++++++++++++++ .../data_stream/260_logs_failure_store.yml | 81 +++++++++++++++++++ .../src/main/resources/logs@settings.json | 5 ++ 3 files changed, 154 insertions(+) create mode 100644 modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml diff --git a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java index 9a3d3bd2fc6e4..d553fa4303540 100644 --- a/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java +++ b/modules/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java @@ -16,6 +16,7 @@ import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -740,6 +741,73 @@ public void testIgnoreDynamicBeyondLimit() throws Exception { assertThat(ignored.stream().filter(i -> i.startsWith("field") == false).toList(), empty()); } + @SuppressWarnings("unchecked") + public void testFailureStoreWithInvalidFieldType() throws Exception { + String dataStreamName = "logs-app-with-failure-store"; + createDataStream(client, dataStreamName); + + indexDoc(client, dataStreamName, """ + { + "@timestamp": "2023-11-30T12:00:00Z", + "message": "This is a valid message" + } + """); + + // invalid document (message as an object instead of string) + indexDoc(client, dataStreamName, """ + { + "@timestamp": "2023-11-30T12:01:00Z", + "message": { + "nested": "This should fail because message should be a string" + } + } + """); + + refreshAllIndices(); + + Request dsInfoRequest = new Request("GET", "/_data_stream/" + dataStreamName); + Map dsInfoResponse = entityAsMap(client.performRequest(dsInfoRequest)); + List> dataStreams = (List>) dsInfoResponse.get("data_streams"); + Map dataStream = dataStreams.getFirst(); + Map failureStoreInfo = (Map) dataStream.get("failure_store"); + assertNotNull(failureStoreInfo); + assertThat(failureStoreInfo.get("enabled"), is(true)); + List> failureIndices = (List>) failureStoreInfo.get("indices"); + + assertThat(failureIndices, not(empty())); + String failureIndex = (String) failureIndices.getFirst().get("index_name"); + assertThat(failureIndex, matchesRegex("\\.fs-" + dataStreamName + "-.*")); + + // query the failure store index + Request failureStoreQuery = new Request("GET", "/" + failureIndex + "/_search"); + failureStoreQuery.setJsonEntity(""" + { + "query": { + "match_all": {} + } + } + """); + Map failureStoreResponse = entityAsMap(client.performRequest(failureStoreQuery)); + Map hits = (Map) failureStoreResponse.get("hits"); + List> hitsList = (List>) hits.get("hits"); + + // Verify the failed document is in the failure store + assertThat(hitsList.size(), is(1)); + Map failedDoc = (Map) hitsList.getFirst().get("_source"); + Map document = (Map) failedDoc.get("document"); + assertNotNull(document); + Map source = (Map) document.get("source"); + assertNotNull(source); + Map message = (Map) source.get("message"); + assertNotNull(message); + assertThat(message.get("nested"), equalTo("This should fail because message should be a string")); + Map error = (Map) failedDoc.get("error"); + assertNotNull(error); + assertEquals("document_parsing_exception", error.get("type")); + String errorMessage = (String) error.get("message"); + assertThat(errorMessage, containsString("failed to parse field [message] of type [match_only_text] in document with id")); + } + @Override protected String indexTemplateName() { return "logs"; diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml new file mode 100644 index 0000000000000..cd69f40ccbcf6 --- /dev/null +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml @@ -0,0 +1,81 @@ +--- +setup: + - requires: + cluster_features: [ "gte_v9.1.99" ] + reason: "failure store became enabled by default for log data streams in 9.2.0" + + - do: + indices.create_data_stream: + name: logs-app-default +--- +teardown: + - do: + indices.delete_data_stream: + name: logs-app-default + ignore: 404 + +--- +"Test logs-*-* data streams have failure store enabled by default": + # index a valid document (string message) + - do: + index: + index: logs-app-default + refresh: true + body: + '@timestamp': '2023-01-01T12:00:00Z' + host: + name: 'server-01' + severity: 'INFO' + message: "Application started successfully" + - match: { result: created } + + - do: + indices.get_data_stream: + name: logs-app-default + - match: { data_streams.0.name: logs-app-default } + - length: { data_streams.0.indices: 1 } + - set: { data_streams.0.indices.0.index_name: idx0name } + - match: { data_streams.0.failure_store.enabled: true } + - length: { data_streams.0.failure_store.indices: 0 } + + # index a document with (object message, causing a mapping conflict) + - do: + index: + index: logs-app-default + refresh: true + body: + '@timestamp': '2023-01-01T12:01:00Z' + host: + name: 'server-02' + severity: 'ERROR' + message: + struct: + value: 42 + - match: { result: 'created' } + - match: { failure_store: used} + + - do: + indices.get_data_stream: + name: logs-app-default + - length: { data_streams.0.failure_store.indices: 1 } + - set: { data_streams.0.failure_store.indices.0.index_name: fs0name } + + - do: + search: + index: $idx0name + body: + query: + match_all: {} + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.severity: "INFO" } + - match: { hits.hits.0._source.message: "Application started successfully" } + + - do: + search: + index: $fs0name + body: + query: + match_all: {} + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.document.source.message.struct.value: 42 } + - match: { hits.hits.0._source.error.type: "document_parsing_exception" } diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json index ca2659b8d8dea..898e1b88cc632 100644 --- a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json +++ b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json @@ -14,6 +14,11 @@ }, "default_pipeline": "logs@default-pipeline" } + }, + "data_stream_options": { + "failure_store": { + "enabled": true + } } }, "_meta": { From 7c53ce757a48190299437389ec2698f78cafdcea Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 15 Jul 2025 06:39:20 +0300 Subject: [PATCH 2/6] Update docs/changelog/131261.yaml --- docs/changelog/131261.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/131261.yaml diff --git a/docs/changelog/131261.yaml b/docs/changelog/131261.yaml new file mode 100644 index 0000000000000..18795d0b9e8d1 --- /dev/null +++ b/docs/changelog/131261.yaml @@ -0,0 +1,6 @@ +pr: 131261 +summary: Enable failure store for log data streams +area: Data streams +type: feature +issues: + - 131105 From 011f8629b7cc5399b09825493bc95b645a0fd568 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Tue, 15 Jul 2025 07:50:37 +0300 Subject: [PATCH 3/6] Adjusting tests --- .../xpack/core/DataStreamRestIT.java | 16 ++++++++-------- .../rest-api-spec/test/stack/10_basic.yml | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java index c4c8d0c460afc..3c380fa546c8e 100644 --- a/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java +++ b/x-pack/plugin/core/src/javaRestTest/java/org/elasticsearch/xpack/core/DataStreamRestIT.java @@ -70,20 +70,20 @@ public void testDSXpackUsage() throws Exception { assertThat(failureStoreStats.get("effectively_enabled_count"), equalTo(0)); assertThat(failureStoreStats.get("failure_indices_count"), equalTo(0)); assertBusy(() -> { - Map logsTemplate = (Map) ((List) getLocation("/_index_template/logs").get("index_templates")).get(0); - assertThat(logsTemplate, notNullValue()); - assertThat(logsTemplate.get("name"), equalTo("logs")); - assertThat(((Map) logsTemplate.get("index_template")).get("data_stream"), notNullValue()); + Map syntheticsTemplate = (Map) ((List) getLocation("/_index_template/synthetics").get("index_templates")).get(0); + assertThat(syntheticsTemplate, notNullValue()); + assertThat(syntheticsTemplate.get("name"), equalTo("synthetics")); + assertThat(((Map) syntheticsTemplate.get("index_template")).get("data_stream"), notNullValue()); }); putFailureStoreTemplate(); // Create a data stream - Request indexRequest = new Request("POST", "/logs-mysql-default/_doc"); + Request indexRequest = new Request("POST", "/synthetics-myapp-default/_doc"); indexRequest.setJsonEntity("{\"@timestamp\": \"2020-01-01\"}"); client().performRequest(indexRequest); // Roll over the data stream - Request rollover = new Request("POST", "/logs-mysql-default/_rollover"); + Request rollover = new Request("POST", "/synthetics-myapp-default/_rollover"); client().performRequest(rollover); // Create failure store data stream @@ -105,10 +105,10 @@ public void testDSXpackUsage() throws Exception { assertThat(failureStoreStats.get("effectively_enabled_count"), equalTo(1)); assertThat(failureStoreStats.get("failure_indices_count"), equalTo(1)); - // Enable the failure store for logs-mysql-default using the cluster setting... + // Enable the failure store for synthetics-myapp-default using the cluster setting... updateClusterSettings( Settings.builder() - .put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "logs-mysql-default") + .put(DataStreamFailureStoreSettings.DATA_STREAM_FAILURE_STORED_ENABLED_SETTING.getKey(), "synthetics-myapp-default") .build() ); // ...and assert that it counts towards effectively_enabled_count but not explicitly_enabled_count: diff --git a/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml b/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml index 3b8bfa9dfb7c8..8d6bf14e419a0 100644 --- a/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml +++ b/x-pack/plugin/stack/src/yamlRestTest/resources/rest-api-spec/test/stack/10_basic.yml @@ -276,7 +276,6 @@ setup: data_stream.namespace: "namespace1" - do: - catch: bad_request index: index: logs-dataset0-namespace1 body: @@ -284,6 +283,7 @@ setup: data_stream.type: "metrics" data_stream.dataset: "dataset0" data_stream.namespace: "namespace1" + - match: { failure_store: used } - do: catch: bad_request From cac9c821f2c91b272af991e222ce4f4207d737af Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Wed, 16 Jul 2025 10:15:41 +0300 Subject: [PATCH 4/6] Update docs/changelog/131261.yaml Co-authored-by: Lee Hinman --- docs/changelog/131261.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/131261.yaml b/docs/changelog/131261.yaml index 18795d0b9e8d1..a45dffc1db366 100644 --- a/docs/changelog/131261.yaml +++ b/docs/changelog/131261.yaml @@ -1,5 +1,5 @@ pr: 131261 -summary: Enable failure store for log data streams +summary: Enable failure store for log-*-* data streams area: Data streams type: feature issues: From 71a4898feef86aeb15f002a5325c87e19f065bda Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Wed, 16 Jul 2025 10:18:59 +0300 Subject: [PATCH 5/6] Bump StackTemplateRegistry#REGISTRY_VERSION --- .../org/elasticsearch/xpack/stack/StackTemplateRegistry.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java index a2f8dd91176e2..bd1178937b463 100644 --- a/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java +++ b/x-pack/plugin/stack/src/main/java/org/elasticsearch/xpack/stack/StackTemplateRegistry.java @@ -37,7 +37,7 @@ public class StackTemplateRegistry extends IndexTemplateRegistry { // The stack template registry version. This number must be incremented when we make changes // to built-in templates. - public static final int REGISTRY_VERSION = 16; + public static final int REGISTRY_VERSION = 17; public static final String TEMPLATE_VERSION_VARIABLE = "xpack.stack.template.version"; public static final Setting STACK_TEMPLATES_ENABLED = Setting.boolSetting( From af5a001ae08da86a86568c8a31f6a452c1729714 Mon Sep 17 00:00:00 2001 From: eyalkoren <41850454+eyalkoren@users.noreply.github.com> Date: Wed, 16 Jul 2025 10:29:47 +0300 Subject: [PATCH 6/6] Replace explicit indices with API conventions in yaml tests --- .../test/data_stream/260_logs_failure_store.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml index cd69f40ccbcf6..e769529a71974 100644 --- a/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml +++ b/modules/data-streams/src/yamlRestTest/resources/rest-api-spec/test/data_stream/260_logs_failure_store.yml @@ -34,7 +34,6 @@ teardown: name: logs-app-default - match: { data_streams.0.name: logs-app-default } - length: { data_streams.0.indices: 1 } - - set: { data_streams.0.indices.0.index_name: idx0name } - match: { data_streams.0.failure_store.enabled: true } - length: { data_streams.0.failure_store.indices: 0 } @@ -58,11 +57,10 @@ teardown: indices.get_data_stream: name: logs-app-default - length: { data_streams.0.failure_store.indices: 1 } - - set: { data_streams.0.failure_store.indices.0.index_name: fs0name } - do: search: - index: $idx0name + index: logs-app-default::data body: query: match_all: {} @@ -72,7 +70,7 @@ teardown: - do: search: - index: $fs0name + index: logs-app-default::failures body: query: match_all: {}