Skip to content

Commit aae06d3

Browse files
kkewweiakolarkunnu
kkewwei
authored and
akolarkunnu
committed
Retrieve value from DocValues in a flat_object filed (opensearch-project#16802)
1 parent 4f6569a commit aae06d3

File tree

6 files changed

+150
-16
lines changed

6 files changed

+150
-16
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
2424
- Added a precaution to handle extreme date values during sorting to prevent `arithmetic_exception: long overflow` ([#16812](https://github.com/opensearch-project/OpenSearch/pull/16812)).
2525
- Add search replica stats to segment replication stats API ([#16678](https://github.com/opensearch-project/OpenSearch/pull/16678))
2626
- Introduce a setting to disable download of full cluster state from remote on term mismatch([#16798](https://github.com/opensearch-project/OpenSearch/pull/16798/))
27+
- Added ability to retrieve value from DocValues in a flat_object filed([#16802](https://github.com/opensearch-project/OpenSearch/pull/16802))
2728

2829
### Dependencies
2930
- Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))

‎rest-api-spec/src/main/resources/rest-api-spec/test/index/92_flat_object_support_doc_values.yml

Lines changed: 49 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
---
22
# The test setup includes:
3-
# - Create flat_object mapping for flat_object_doc_values_test index
4-
# - Index 9 example documents
5-
# - Search tests about doc_values and index
3+
# - 1.Create flat_object mapping for flat_object_doc_values_test index
4+
# - 2.Index 9 example documents
5+
# - 3.Search tests about doc_values and index
6+
# - 4.Fetch doc_value from flat_object field
67

78
setup:
89
- skip:
@@ -786,3 +787,48 @@ teardown:
786787

787788
- length: { hits.hits: 1 }
788789
- match: { hits.hits.0._source.order: "order8" }
790+
791+
# Stored Fields with exact dot path.
792+
- do:
793+
search:
794+
body: {
795+
_source: false,
796+
query: {
797+
bool: {
798+
must: [
799+
{
800+
term: {
801+
order: "order0"
802+
}
803+
}
804+
]
805+
}
806+
},
807+
stored_fields: "_none_",
808+
docvalue_fields: [ "issue.labels.name","order" ]
809+
}
810+
811+
- length: { hits.hits: 1 }
812+
- match: { hits.hits.0.fields: { "order" : [ "order0" ], "issue.labels.name": [ "abc0" ] } }
813+
814+
- do:
815+
search:
816+
body: {
817+
_source: false,
818+
query: {
819+
bool: {
820+
must: [
821+
{
822+
term: {
823+
order: "order0"
824+
}
825+
}
826+
]
827+
}
828+
},
829+
stored_fields: "_none_",
830+
docvalue_fields: [ "issue.labels.name" ]
831+
}
832+
833+
- length: { hits.hits: 1 }
834+
- match: { hits.hits.0.fields: { "issue.labels.name": [ "abc0" ] } }

‎server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1023,7 +1023,7 @@ public void testDocValueFields() throws Exception {
10231023
.startObject("ip_field")
10241024
.field("type", "ip")
10251025
.endObject()
1026-
.startObject("flat_object_field")
1026+
.startObject("flat_object_field1")
10271027
.field("type", "flat_object")
10281028
.endObject()
10291029
.endObject()
@@ -1050,9 +1050,11 @@ public void testDocValueFields() throws Exception {
10501050
.field("boolean_field", true)
10511051
.field("binary_field", new byte[] { 42, 100 })
10521052
.field("ip_field", "::1")
1053-
.field("flat_object_field")
1053+
.field("flat_object_field1")
10541054
.startObject()
1055+
.field("fooa", "bara")
10551056
.field("foo", "bar")
1057+
.field("foob", "barb")
10561058
.endObject()
10571059
.endObject()
10581060
)
@@ -1075,7 +1077,7 @@ public void testDocValueFields() throws Exception {
10751077
.addDocValueField("boolean_field")
10761078
.addDocValueField("binary_field")
10771079
.addDocValueField("ip_field")
1078-
.addDocValueField("flat_object_field");
1080+
.addDocValueField("flat_object_field1.foo");
10791081
SearchResponse searchResponse = builder.get();
10801082

10811083
assertThat(searchResponse.getHits().getTotalHits().value, equalTo(1L));
@@ -1097,7 +1099,7 @@ public void testDocValueFields() throws Exception {
10971099
"keyword_field",
10981100
"binary_field",
10991101
"ip_field",
1100-
"flat_object_field"
1102+
"flat_object_field1.foo"
11011103
)
11021104
)
11031105
);
@@ -1116,7 +1118,7 @@ public void testDocValueFields() throws Exception {
11161118
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
11171119
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ"));
11181120
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));
1119-
assertThat(searchResponse.getHits().getAt(0).getFields().get("flat_object_field").getValue(), equalTo("flat_object_field.foo"));
1121+
assertThat(searchResponse.getHits().getAt(0).getFields().get("flat_object_field1.foo").getValue(), equalTo("bar"));
11201122

11211123
builder = client().prepareSearch().setQuery(matchAllQuery()).addDocValueField("*field");
11221124
searchResponse = builder.get();
@@ -1139,8 +1141,7 @@ public void testDocValueFields() throws Exception {
11391141
"text_field",
11401142
"keyword_field",
11411143
"binary_field",
1142-
"ip_field",
1143-
"flat_object_field"
1144+
"ip_field"
11441145
)
11451146
)
11461147
);
@@ -1160,7 +1161,6 @@ public void testDocValueFields() throws Exception {
11601161
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
11611162
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ"));
11621163
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));
1163-
assertThat(searchResponse.getHits().getAt(0).getFields().get("flat_object_field").getValue(), equalTo("flat_object_field.foo"));
11641164

11651165
builder = client().prepareSearch()
11661166
.setQuery(matchAllQuery())
@@ -1176,7 +1176,7 @@ public void testDocValueFields() throws Exception {
11761176
.addDocValueField("boolean_field", "use_field_mapping")
11771177
.addDocValueField("binary_field", "use_field_mapping")
11781178
.addDocValueField("ip_field", "use_field_mapping")
1179-
.addDocValueField("flat_object_field", "use_field_mapping");
1179+
.addDocValueField("flat_object_field1.foo", null);
11801180
;
11811181
searchResponse = builder.get();
11821182

@@ -1199,7 +1199,7 @@ public void testDocValueFields() throws Exception {
11991199
"keyword_field",
12001200
"binary_field",
12011201
"ip_field",
1202-
"flat_object_field"
1202+
"flat_object_field1.foo"
12031203
)
12041204
)
12051205
);
@@ -1219,7 +1219,7 @@ public void testDocValueFields() throws Exception {
12191219
assertThat(searchResponse.getHits().getAt(0).getFields().get("keyword_field").getValue(), equalTo("foo"));
12201220
assertThat(searchResponse.getHits().getAt(0).getFields().get("binary_field").getValue(), equalTo("KmQ"));
12211221
assertThat(searchResponse.getHits().getAt(0).getFields().get("ip_field").getValue(), equalTo("::1"));
1222-
assertThat(searchResponse.getHits().getAt(0).getFields().get("flat_object_field").getValue(), equalTo("flat_object_field.foo"));
1222+
assertThat(searchResponse.getHits().getAt(0).getFields().get("flat_object_field1.foo").getValue(), equalTo("bar"));
12231223

12241224
builder = client().prepareSearch()
12251225
.setQuery(matchAllQuery())

‎server/src/main/java/org/opensearch/index/mapper/DocValueFetcher.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
import java.util.List;
4444

4545
import static java.util.Collections.emptyList;
46+
import static org.opensearch.index.mapper.FlatObjectFieldMapper.DOC_VALUE_NO_MATCH;
4647

4748
/**
4849
* Value fetcher that loads from doc values.
@@ -70,7 +71,10 @@ public List<Object> fetchValues(SourceLookup lookup) throws IOException {
7071
}
7172
List<Object> result = new ArrayList<Object>(leaf.docValueCount());
7273
for (int i = 0, count = leaf.docValueCount(); i < count; ++i) {
73-
result.add(leaf.nextValue());
74+
Object value = leaf.nextValue();
75+
if (value != DOC_VALUE_NO_MATCH) {
76+
result.add(value);
77+
}
7478
}
7579
return result;
7680
}

‎server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.opensearch.common.unit.Fuzziness;
2929
import org.opensearch.common.xcontent.JsonToStringXContentParser;
3030
import org.opensearch.core.common.ParsingException;
31+
import org.opensearch.core.common.io.stream.StreamOutput;
3132
import org.opensearch.core.xcontent.DeprecationHandler;
3233
import org.opensearch.core.xcontent.NamedXContentRegistry;
3334
import org.opensearch.core.xcontent.XContentParser;
@@ -36,11 +37,13 @@
3637
import org.opensearch.index.fielddata.plain.SortedSetOrdinalsIndexFieldData;
3738
import org.opensearch.index.mapper.KeywordFieldMapper.KeywordFieldType;
3839
import org.opensearch.index.query.QueryShardContext;
40+
import org.opensearch.search.DocValueFormat;
3941
import org.opensearch.search.aggregations.support.CoreValuesSourceType;
4042
import org.opensearch.search.lookup.SearchLookup;
4143

4244
import java.io.IOException;
4345
import java.io.UncheckedIOException;
46+
import java.time.ZoneId;
4447
import java.util.ArrayList;
4548
import java.util.Collections;
4649
import java.util.Iterator;
@@ -63,6 +66,7 @@
6366
public final class FlatObjectFieldMapper extends DynamicKeyFieldMapper {
6467

6568
public static final String CONTENT_TYPE = "flat_object";
69+
public static final Object DOC_VALUE_NO_MATCH = new Object();
6670

6771
/**
6872
* In flat_object field mapper, field type is similar to keyword field type
@@ -272,7 +276,7 @@ NamedAnalyzer normalizer() {
272276
@Override
273277
public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
274278
failIfNoDocValues();
275-
return new SortedSetOrdinalsIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES);
279+
return new SortedSetOrdinalsIndexFieldData.Builder(valueFieldType().name(), CoreValuesSourceType.BYTES);
276280
}
277281

278282
@Override
@@ -304,6 +308,30 @@ protected String parseSourceValue(Object value) {
304308
};
305309
}
306310

311+
@Override
312+
public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
313+
if (format != null) {
314+
throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support custom formats");
315+
}
316+
if (timeZone != null) {
317+
throw new IllegalArgumentException(
318+
"Field [" + name() + "] of type [" + typeName() + "] does not support custom time zones"
319+
);
320+
}
321+
if (mappedFieldTypeName != null) {
322+
return new FlatObjectDocValueFormat(mappedFieldTypeName + DOT_SYMBOL + name() + EQUAL_SYMBOL);
323+
} else {
324+
throw new IllegalArgumentException(
325+
"Field [" + name() + "] of type [" + typeName() + "] does not support doc_value in root field"
326+
);
327+
}
328+
}
329+
330+
@Override
331+
public boolean isAggregatable() {
332+
return false;
333+
}
334+
307335
@Override
308336
public Object valueForDisplay(Object value) {
309337
if (value == null) {
@@ -530,6 +558,39 @@ public Query wildcardQuery(
530558
return valueFieldType().wildcardQuery(rewriteValue(value), method, caseInsensitve, context);
531559
}
532560

561+
/**
562+
* A doc_value formatter for flat_object field.
563+
*/
564+
public class FlatObjectDocValueFormat implements DocValueFormat {
565+
private static final String NAME = "flat_object";
566+
private final String prefix;
567+
568+
public FlatObjectDocValueFormat(String prefix) {
569+
this.prefix = prefix;
570+
}
571+
572+
@Override
573+
public String getWriteableName() {
574+
return NAME;
575+
}
576+
577+
@Override
578+
public void writeTo(StreamOutput out) {}
579+
580+
@Override
581+
public Object format(BytesRef value) {
582+
String parsedValue = inputToString(value);
583+
if (parsedValue.startsWith(prefix) == false) {
584+
return DOC_VALUE_NO_MATCH;
585+
}
586+
return parsedValue.substring(prefix.length());
587+
}
588+
589+
@Override
590+
public BytesRef parseBytesRef(String value) {
591+
return new BytesRef((String) valueFieldType.rewriteForDocValue(rewriteValue(value)));
592+
}
593+
}
533594
}
534595

535596
private final ValueFieldMapper valueFieldMapper;

‎server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldMapperTests.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.opensearch.core.xcontent.ToXContent;
2222
import org.opensearch.core.xcontent.XContentBuilder;
2323
import org.opensearch.index.query.QueryShardContext;
24+
import org.opensearch.search.DocValueFormat;
2425

2526
import java.io.IOException;
2627

@@ -397,6 +398,27 @@ public void testDeduplicationValue() throws IOException {
397398
assertEquals(new BytesRef("field.labels=3"), fieldValueAndPaths[4].binaryValue());
398399
}
399400

401+
public void testFetchDocValues() throws IOException {
402+
MapperService mapperService = createMapperService(fieldMapping(b -> b.field("type", "flat_object")));
403+
{
404+
// test valueWithPathField
405+
MappedFieldType ft = mapperService.fieldType("field.name");
406+
DocValueFormat format = ft.docValueFormat(null, null);
407+
String storedValue = "field.field.name=1234";
408+
409+
Object object = format.format(new BytesRef(storedValue));
410+
assertEquals("1234", object);
411+
}
412+
413+
{
414+
// test valueField
415+
MappedFieldType ft = mapperService.fieldType("field");
416+
Throwable throwable = assertThrows(IllegalArgumentException.class, () -> ft.docValueFormat(null, null));
417+
assertEquals("Field [field] of type [flat_object] does not support doc_value in root field", throwable.getMessage());
418+
}
419+
420+
}
421+
400422
@Override
401423
protected void registerParameters(ParameterChecker checker) throws IOException {
402424
// In the future we will want to make sure parameter updates are covered.

0 commit comments

Comments
 (0)