From 374cee8110f931702c52189d3355ca097d4d1fc0 Mon Sep 17 00:00:00 2001 From: m-Bilal Date: Fri, 18 Apr 2025 19:47:21 +0530 Subject: [PATCH 1/2] set track_total_hits to true in aggregation queries --- connector/query.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/connector/query.go b/connector/query.go index 27e9e38..20d4e98 100644 --- a/connector/query.go +++ b/connector/query.go @@ -190,13 +190,17 @@ func prepareElasticsearchQuery(ctx context.Context, request *schema.QueryRequest } if len(aggs) != 0 { query["aggs"] = aggs - - // set query size to 0 if aggregation is present - // this is because, by default, an aggregation query returns both the aggregation result and the hit documents - // we only want the aggregation result - // more reading: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html#return-only-agg-results - query["size"] = 0 } + + // set query size to 0 if aggregation is present + // this is because, by default, an aggregation query returns both the aggregation result and the hit documents + // we only want the aggregation result + // more reading: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html#return-only-agg-results + query["size"] = 0 + + // set track_total_hits to true if aggregation is present + // this is because, by default, a count query will only return 10,000 hits + query["track_total_hits"] = true } span.AddEvent("prepare_filter_query") From 3eb5f74f23a8654b1ca99ded7f8c12e7925a1330 Mon Sep 17 00:00:00 2001 From: m-Bilal Date: Fri, 18 Apr 2025 19:47:44 +0530 Subject: [PATCH 2/2] update tests --- .../query_tests/payments/cardinality_aggregation/want.json | 3 ++- .../query_tests/payments/count_aggregation/want.json | 3 ++- .../query_tests/payments/count_distinct_aggregation/want.json | 3 ++- .../query_tests/payments/float_aggregations/want.json | 3 ++- .../payments/float_aggregations_with_range/want.json | 3 ++- .../query_tests/payments/multiple_aggregations/want.json | 3 ++- .../payments/multiple_aggregations_with_where_clause/want.json | 3 ++- .../query_tests/payments/nested_aggregations/want.json | 3 ++- .../payments/subfield_cardinality_aggregation/want.json | 3 ++- 9 files changed, 18 insertions(+), 9 deletions(-) diff --git a/testdata/unit_tests/query_tests/payments/cardinality_aggregation/want.json b/testdata/unit_tests/query_tests/payments/cardinality_aggregation/want.json index 6bfea1a..dea4d76 100644 --- a/testdata/unit_tests/query_tests/payments/cardinality_aggregation/want.json +++ b/testdata/unit_tests/query_tests/payments/cardinality_aggregation/want.json @@ -11,5 +11,6 @@ } } }, - "size": 0 + "size": 0, + "track_total_hits": true } \ No newline at end of file diff --git a/testdata/unit_tests/query_tests/payments/count_aggregation/want.json b/testdata/unit_tests/query_tests/payments/count_aggregation/want.json index 0f65c34..b59bbf0 100644 --- a/testdata/unit_tests/query_tests/payments/count_aggregation/want.json +++ b/testdata/unit_tests/query_tests/payments/count_aggregation/want.json @@ -4,5 +4,6 @@ "*" ] }, - "size": 10000 + "size": 0, + "track_total_hits": true } \ No newline at end of file diff --git a/testdata/unit_tests/query_tests/payments/count_distinct_aggregation/want.json b/testdata/unit_tests/query_tests/payments/count_distinct_aggregation/want.json index 3882367..c506a60 100644 --- a/testdata/unit_tests/query_tests/payments/count_distinct_aggregation/want.json +++ b/testdata/unit_tests/query_tests/payments/count_distinct_aggregation/want.json @@ -11,5 +11,6 @@ } } }, - "size": 0 + "size": 0, + "track_total_hits": true } \ No newline at end of file diff --git a/testdata/unit_tests/query_tests/payments/float_aggregations/want.json b/testdata/unit_tests/query_tests/payments/float_aggregations/want.json index b5c265c..8d0f8f6 100644 --- a/testdata/unit_tests/query_tests/payments/float_aggregations/want.json +++ b/testdata/unit_tests/query_tests/payments/float_aggregations/want.json @@ -53,5 +53,6 @@ } } }, - "size": 0 + "size": 0, + "track_total_hits": true } \ No newline at end of file diff --git a/testdata/unit_tests/query_tests/payments/float_aggregations_with_range/want.json b/testdata/unit_tests/query_tests/payments/float_aggregations_with_range/want.json index 1c4ee20..5fadb99 100644 --- a/testdata/unit_tests/query_tests/payments/float_aggregations_with_range/want.json +++ b/testdata/unit_tests/query_tests/payments/float_aggregations_with_range/want.json @@ -61,5 +61,6 @@ } } }, - "size": 0 + "size": 0, + "track_total_hits": true } \ No newline at end of file diff --git a/testdata/unit_tests/query_tests/payments/multiple_aggregations/want.json b/testdata/unit_tests/query_tests/payments/multiple_aggregations/want.json index 0ec9225..a3af90c 100644 --- a/testdata/unit_tests/query_tests/payments/multiple_aggregations/want.json +++ b/testdata/unit_tests/query_tests/payments/multiple_aggregations/want.json @@ -60,5 +60,6 @@ } } }, - "size": 0 + "size": 0, + "track_total_hits": true } \ No newline at end of file diff --git a/testdata/unit_tests/query_tests/payments/multiple_aggregations_with_where_clause/want.json b/testdata/unit_tests/query_tests/payments/multiple_aggregations_with_where_clause/want.json index 039f02c..154ff1d 100644 --- a/testdata/unit_tests/query_tests/payments/multiple_aggregations_with_where_clause/want.json +++ b/testdata/unit_tests/query_tests/payments/multiple_aggregations_with_where_clause/want.json @@ -65,5 +65,6 @@ "customer_id": "CUST" } }, - "size": 0 + "size": 0, + "track_total_hits": true } \ No newline at end of file diff --git a/testdata/unit_tests/query_tests/payments/nested_aggregations/want.json b/testdata/unit_tests/query_tests/payments/nested_aggregations/want.json index 03d14f6..8a68c1c 100644 --- a/testdata/unit_tests/query_tests/payments/nested_aggregations/want.json +++ b/testdata/unit_tests/query_tests/payments/nested_aggregations/want.json @@ -31,5 +31,6 @@ } } }, - "size": 0 + "size": 0, + "track_total_hits": true } \ No newline at end of file diff --git a/testdata/unit_tests/query_tests/payments/subfield_cardinality_aggregation/want.json b/testdata/unit_tests/query_tests/payments/subfield_cardinality_aggregation/want.json index 56770bd..8c11cf0 100644 --- a/testdata/unit_tests/query_tests/payments/subfield_cardinality_aggregation/want.json +++ b/testdata/unit_tests/query_tests/payments/subfield_cardinality_aggregation/want.json @@ -11,5 +11,6 @@ } } }, - "size": 0 + "size": 0, + "track_total_hits": true } \ No newline at end of file