Skip to content

Commit d2e90c6

Browse files
github-actions[bot]mkhludnev
github-actions[bot]
and
mkhludnev
committed
Search dv only IP masks (#16628)
* search dv only ip masks Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> * search dv only ip masks: changes Signed-off-by: Mikhail Khludnev <mkhl@apache.org> * drop fancy closures Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> * drop fancy closures. one more Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> * fix unit tests; add some more dvOnly Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> * drop skipping dvOnly in 2.x Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> * drop redundant brackets Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> * extract conditions Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> * asserts Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> * spotless apply Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> * bring back skip before Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> * combine asserts Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> * inline, copy-paste Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> --------- Signed-off-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> Signed-off-by: Mikhail Khludnev <mkhl@apache.org> Co-authored-by: mikhail-khludnev <mikhail_khludnev@rntgroup.com> (cherry picked from commit d4d70d8) Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
1 parent df175d1 commit d2e90c6

File tree

4 files changed

+240
-46
lines changed

4 files changed

+240
-46
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
4444
- Support retrieving doc values of unsigned long field ([#16543](https://github.com/opensearch-project/OpenSearch/pull/16543))
4545
- Fix rollover alias supports restored searchable snapshot index([#16483](https://github.com/opensearch-project/OpenSearch/pull/16483))
4646
- Fix permissions error on scripted query against remote snapshot ([#16544](https://github.com/opensearch-project/OpenSearch/pull/16544))
47+
- Fix `doc_values` only (`index:false`) IP field searching for masks ([#16628](https://github.com/opensearch-project/OpenSearch/pull/16628))
4748

4849
### Security
4950

‎rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml

Lines changed: 90 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -454,6 +454,28 @@ setup:
454454

455455
- match: { hits.total: 2 }
456456

457+
- do:
458+
search:
459+
rest_total_hits_as_int: true
460+
index: test-iodvq
461+
body:
462+
query:
463+
term:
464+
ip_field: "192.168.0.1/24"
465+
466+
- match: { hits.total: 3 }
467+
468+
- do:
469+
search:
470+
rest_total_hits_as_int: true
471+
index: test-iodvq
472+
body:
473+
query:
474+
term:
475+
ip_field: "192.168.0.1/31"
476+
477+
- match: { hits.total: 1 }
478+
457479
- do:
458480
search:
459481
rest_total_hits_as_int: true
@@ -992,6 +1014,28 @@ setup:
9921014

9931015
- match: { hits.total: 2 }
9941016

1017+
- do:
1018+
search:
1019+
rest_total_hits_as_int: true
1020+
index: test-index
1021+
body:
1022+
query:
1023+
term:
1024+
ip_field: "192.168.0.1/24"
1025+
1026+
- match: { hits.total: 3 }
1027+
1028+
- do:
1029+
search:
1030+
rest_total_hits_as_int: true
1031+
index: test-index
1032+
body:
1033+
query:
1034+
term:
1035+
ip_field: "192.168.0.1/31"
1036+
1037+
- match: { hits.total: 1 }
1038+
9951039
- do:
9961040
search:
9971041
rest_total_hits_as_int: true
@@ -1082,8 +1126,8 @@ setup:
10821126
"search on fields with only doc_values enabled":
10831127
- skip:
10841128
features: [ "headers" ]
1085-
version: " - 2.99.99"
1086-
reason: "searching with only doc_values was added in 3.0.0"
1129+
version: " - 2.18.99"
1130+
reason: "searching with only doc_values was finally added in 2.19.0"
10871131
- do:
10881132
indices.create:
10891133
index: test-doc-values
@@ -1377,6 +1421,28 @@ setup:
13771421

13781422
- match: { hits.total: 2 }
13791423

1424+
- do:
1425+
search:
1426+
rest_total_hits_as_int: true
1427+
index: test-doc-values
1428+
body:
1429+
query:
1430+
terms:
1431+
ip_field: ["192.168.0.1", "192.168.0.2"]
1432+
1433+
- match: { hits.total: 2 }
1434+
1435+
- do:
1436+
search:
1437+
rest_total_hits_as_int: true
1438+
index: test-doc-values
1439+
body:
1440+
query:
1441+
terms:
1442+
ip_field: ["192.168.0.1/31", "192.168.0.3"]
1443+
1444+
- match: { hits.total: 2 }
1445+
13801446
- do:
13811447
search:
13821448
rest_total_hits_as_int: true
@@ -1521,6 +1587,28 @@ setup:
15211587

15221588
- match: { hits.total: 2 }
15231589

1590+
- do:
1591+
search:
1592+
rest_total_hits_as_int: true
1593+
index: test-doc-values
1594+
body:
1595+
query:
1596+
term:
1597+
ip_field: "192.168.0.1/31"
1598+
1599+
- match: { hits.total: 1 }
1600+
1601+
- do:
1602+
search:
1603+
rest_total_hits_as_int: true
1604+
index: test-doc-values
1605+
body:
1606+
query:
1607+
term:
1608+
ip_field: "192.168.0.1/24"
1609+
1610+
- match: { hits.total: 3 }
1611+
15241612
- do:
15251613
search:
15261614
rest_total_hits_as_int: true

‎server/src/main/java/org/opensearch/index/mapper/IpFieldMapper.java

Lines changed: 46 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import java.util.Map;
6565
import java.util.function.BiFunction;
6666
import java.util.function.Supplier;
67+
import java.util.stream.Collectors;
6768

6869
/**
6970
* A {@link FieldMapper} for ip addresses.
@@ -225,42 +226,37 @@ protected Object parseSourceValue(Object value) {
225226
@Override
226227
public Query termQuery(Object value, @Nullable QueryShardContext context) {
227228
failIfNotIndexedAndNoDocValues();
228-
Query query;
229+
final PointRangeQuery pointQuery;
229230
if (value instanceof InetAddress) {
230-
query = InetAddressPoint.newExactQuery(name(), (InetAddress) value);
231+
pointQuery = (PointRangeQuery) InetAddressPoint.newExactQuery(name(), (InetAddress) value);
231232
} else {
232233
if (value instanceof BytesRef) {
233234
value = ((BytesRef) value).utf8ToString();
234235
}
235236
String term = value.toString();
236237
if (term.contains("/")) {
237238
final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
238-
query = InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
239+
pointQuery = (PointRangeQuery) InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
239240
} else {
240241
InetAddress address = InetAddresses.forString(term);
241-
query = InetAddressPoint.newExactQuery(name(), address);
242+
pointQuery = (PointRangeQuery) InetAddressPoint.newExactQuery(name(), address);
242243
}
243244
}
244-
if (isSearchable() && hasDocValues()) {
245-
String term = value.toString();
246-
if (term.contains("/")) {
247-
final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
248-
return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
249-
}
250-
return new IndexOrDocValuesQuery(
251-
query,
252-
SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint()))
245+
Query dvQuery = null;
246+
if (hasDocValues()) {
247+
dvQuery = SortedSetDocValuesField.newSlowRangeQuery(
248+
name(),
249+
new BytesRef(pointQuery.getLowerPoint()),
250+
new BytesRef(pointQuery.getUpperPoint()),
251+
true,
252+
true
253253
);
254254
}
255-
if (hasDocValues()) {
256-
String term = value.toString();
257-
if (term.contains("/")) {
258-
final Tuple<InetAddress, Integer> cidr = InetAddresses.parseCidr(term);
259-
return InetAddressPoint.newPrefixQuery(name(), cidr.v1(), cidr.v2());
260-
}
261-
return SortedSetDocValuesField.newSlowExactQuery(name(), new BytesRef(((PointRangeQuery) query).getLowerPoint()));
255+
if (isSearchable() && hasDocValues()) {
256+
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
257+
} else {
258+
return isSearchable() ? pointQuery : dvQuery;
262259
}
263-
return query;
264260
}
265261

266262
@Override
@@ -285,36 +281,46 @@ public Query termsQuery(List<?> values, QueryShardContext context) {
285281
}
286282
addresses[i++] = address;
287283
}
288-
return InetAddressPoint.newSetQuery(name(), addresses);
284+
Query dvQuery = null;
285+
if (hasDocValues()) {
286+
List<BytesRef> bytesRefs = Arrays.stream(addresses)
287+
.distinct()
288+
.map(InetAddressPoint::encode)
289+
.map(BytesRef::new)
290+
.collect(Collectors.<BytesRef>toList());
291+
dvQuery = SortedSetDocValuesField.newSlowSetQuery(name(), bytesRefs);
292+
}
293+
Query pointQuery = null;
294+
if (isSearchable()) {
295+
pointQuery = InetAddressPoint.newSetQuery(name(), addresses);
296+
}
297+
if (isSearchable() && hasDocValues()) {
298+
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
299+
} else {
300+
return isSearchable() ? pointQuery : dvQuery;
301+
}
289302
}
290303

291304
@Override
292305
public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
293306
failIfNotIndexedAndNoDocValues();
294307
return rangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, (lower, upper) -> {
295-
Query query = InetAddressPoint.newRangeQuery(name(), lower, upper);
296-
if (isSearchable() && hasDocValues()) {
297-
return new IndexOrDocValuesQuery(
298-
query,
299-
SortedSetDocValuesField.newSlowRangeQuery(
300-
((PointRangeQuery) query).getField(),
301-
new BytesRef(((PointRangeQuery) query).getLowerPoint()),
302-
new BytesRef(((PointRangeQuery) query).getUpperPoint()),
303-
true,
304-
true
305-
)
306-
);
307-
}
308+
PointRangeQuery pointQuery = (PointRangeQuery) InetAddressPoint.newRangeQuery(name(), lower, upper);
309+
Query dvQuery = null;
308310
if (hasDocValues()) {
309-
return SortedSetDocValuesField.newSlowRangeQuery(
310-
((PointRangeQuery) query).getField(),
311-
new BytesRef(((PointRangeQuery) query).getLowerPoint()),
312-
new BytesRef(((PointRangeQuery) query).getUpperPoint()),
311+
dvQuery = SortedSetDocValuesField.newSlowRangeQuery(
312+
pointQuery.getField(),
313+
new BytesRef(pointQuery.getLowerPoint()),
314+
new BytesRef(pointQuery.getUpperPoint()),
313315
true,
314316
true
315317
);
316318
}
317-
return query;
319+
if (isSearchable() && hasDocValues()) {
320+
return new IndexOrDocValuesQuery(pointQuery, dvQuery);
321+
} else {
322+
return isSearchable() ? pointQuery : dvQuery;
323+
}
318324
});
319325
}
320326

0 commit comments

Comments
 (0)