Skip to content

Commit 07a39d6

Browse files
navneet1vnavneet1v
committed
Add capability to disable source recovery_source for an index
Signed-off-by: Navneet Verma <navneev@amazon.com>
1 parent 6b6a0a2 commit 07a39d6

File tree

2 files changed

+193
-13
lines changed

2 files changed

+193
-13
lines changed

‎server/src/main/java/org/opensearch/index/mapper/SourceFieldMapper.java

Lines changed: 48 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ public class SourceFieldMapper extends MetadataFieldMapper {
7474

7575
public static final String CONTENT_TYPE = "_source";
7676
private final Function<Map<String, ?>, Map<String, Object>> filter;
77+
private final Function<Map<String, ?>, Map<String, Object>> recoverySourceFilter;
7778

7879
/**
7980
* Default parameters for source fields
@@ -119,21 +120,34 @@ public static class Builder extends MetadataFieldMapper.Builder {
119120
Collections.emptyList()
120121
);
121122

123+
private final Parameter<Boolean> recoverySourceEnabled = Parameter.boolParam("recovery_source_enabled",
124+
false, m -> toType(m).recoverySourceEnabled, Defaults.ENABLED);
125+
126+
private final Parameter<List<String>> recoverySourceIncludes = Parameter.stringArrayParam(
127+
"recovery_source_includes", false, m->Arrays.asList(toType(m).recoverySourceIncludes), Collections.emptyList());
128+
129+
private final Parameter<List<String>> recoverySourceExcludes = Parameter.stringArrayParam(
130+
"recovery_source_excludes", false, m->Arrays.asList(toType(m).recoverySourceExcludes),
131+
Collections.emptyList());
132+
122133
public Builder() {
123134
super(Defaults.NAME);
124135
}
125136

126137
@Override
127138
protected List<Parameter<?>> getParameters() {
128-
return Arrays.asList(enabled, includes, excludes);
139+
return Arrays.asList(enabled, includes, excludes, recoverySourceEnabled, recoverySourceIncludes, recoverySourceExcludes);
129140
}
130141

131142
@Override
132143
public SourceFieldMapper build(BuilderContext context) {
133144
return new SourceFieldMapper(
134145
enabled.getValue(),
135146
includes.getValue().toArray(new String[0]),
136-
excludes.getValue().toArray(new String[0])
147+
excludes.getValue().toArray(new String[0]),
148+
recoverySourceEnabled.getValue(),
149+
recoverySourceIncludes.getValue().toArray(new String[0]),
150+
recoverySourceExcludes.getValue().toArray(new String[0])
137151
);
138152
}
139153
}
@@ -173,24 +187,36 @@ public Query termQuery(Object value, QueryShardContext context) {
173187
}
174188

175189
private final boolean enabled;
190+
private final boolean recoverySourceEnabled;
176191
/** indicates whether the source will always exist and be complete, for use by features like the update API */
177192
private final boolean complete;
178193

179194
private final String[] includes;
180195
private final String[] excludes;
196+
private final String[] recoverySourceIncludes;
197+
private final String[] recoverySourceExcludes;
181198

182199
private SourceFieldMapper() {
183-
this(Defaults.ENABLED, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY);
200+
this(Defaults.ENABLED, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY, Defaults.ENABLED, Strings.EMPTY_ARRAY, Strings.EMPTY_ARRAY);
184201
}
185202

186-
private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes) {
203+
private SourceFieldMapper(boolean enabled, String[] includes, String[] excludes, boolean recoverySourceEnabled,
204+
String[] recoverySourceIncludes, String[] recoverySourceExcludes) {
187205
super(new SourceFieldType(enabled));
188206
this.enabled = enabled;
189207
this.includes = includes;
190208
this.excludes = excludes;
191209
final boolean filtered = CollectionUtils.isEmpty(includes) == false || CollectionUtils.isEmpty(excludes) == false;
192210
this.filter = enabled && filtered ? XContentMapValues.filter(includes, excludes) : null;
193211
this.complete = enabled && CollectionUtils.isEmpty(includes) && CollectionUtils.isEmpty(excludes);
212+
213+
// Set parameters for recovery source
214+
this.recoverySourceEnabled = recoverySourceEnabled;
215+
this.recoverySourceIncludes = recoverySourceIncludes;
216+
this.recoverySourceExcludes = recoverySourceExcludes;
217+
final boolean recoverySourcefiltered =
218+
CollectionUtils.isEmpty(recoverySourceIncludes) == false || CollectionUtils.isEmpty(recoverySourceExcludes) == false;
219+
this.recoverySourceFilter = this.recoverySourceEnabled && recoverySourcefiltered ? XContentMapValues.filter(recoverySourceIncludes, recoverySourceExcludes) : null;
194220
}
195221

196222
public boolean enabled() {
@@ -212,22 +238,32 @@ public void preParse(ParseContext context) throws IOException {
212238
context.doc().add(new StoredField(fieldType().name(), ref.bytes, ref.offset, ref.length));
213239
}
214240

215-
if (originalSource != null && adaptedSource != originalSource) {
216-
// if we omitted source or modified it we add the _recovery_source to ensure we have it for ops based recovery
217-
BytesRef ref = originalSource.toBytesRef();
218-
context.doc().add(new StoredField(RECOVERY_SOURCE_NAME, ref.bytes, ref.offset, ref.length));
219-
context.doc().add(new NumericDocValuesField(RECOVERY_SOURCE_NAME, 1));
241+
if(recoverySourceEnabled) {
242+
if (originalSource != null && adaptedSource != originalSource) {
243+
final BytesReference adaptedRecoverySource = applyFilters(originalSource, contentType,
244+
recoverySourceEnabled, recoverySourceFilter);
245+
// if we omitted source or modified it we add the _recovery_source to ensure we have it for ops based recovery
246+
BytesRef ref = adaptedRecoverySource.toBytesRef();
247+
context.doc().add(new StoredField(RECOVERY_SOURCE_NAME, ref.bytes, ref.offset, ref.length));
248+
context.doc().add(new NumericDocValuesField(RECOVERY_SOURCE_NAME, 1));
249+
}
220250
}
221251
}
222252

223253
@Nullable
224254
public BytesReference applyFilters(@Nullable BytesReference originalSource, @Nullable MediaType contentType) throws IOException {
225-
if (enabled && originalSource != null) {
255+
return applyFilters(originalSource, contentType, enabled, filter);
256+
}
257+
258+
@Nullable
259+
private BytesReference applyFilters(@Nullable BytesReference originalSource, @Nullable MediaType contentType,
260+
boolean isProvidedSourceEnabled, @Nullable final Function<Map<String, ?>, Map<String, Object>> filters) throws IOException {
261+
if (isProvidedSourceEnabled && originalSource != null) {
226262
// Percolate and tv APIs may not set the source and that is ok, because these APIs will not index any data
227-
if (filter != null) {
263+
if (filters != null) {
228264
// we don't update the context source if we filter, we want to keep it as is...
229265
Tuple<? extends MediaType, Map<String, Object>> mapTuple = XContentHelper.convertToMap(originalSource, true, contentType);
230-
Map<String, Object> filteredSource = filter.apply(mapTuple.v2());
266+
Map<String, Object> filteredSource = filters.apply(mapTuple.v2());
231267
BytesStreamOutput bStream = new BytesStreamOutput();
232268
MediaType actualContentType = mapTuple.v1();
233269
XContentBuilder builder = MediaTypeRegistry.contentBuilder(actualContentType, bStream).map(filteredSource);

‎server/src/test/java/org/opensearch/index/mapper/SourceFieldMapperTests.java

Lines changed: 145 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,8 @@ public void testNoFormat() throws Exception {
9090
XContentType.SMILE
9191
)
9292
);
93-
93+
final IndexableField recoverySourceIndexableField = doc.rootDoc().getField("_recovery_source");
94+
assertNull(recoverySourceIndexableField);
9495
assertThat(MediaTypeRegistry.xContentType(doc.source()), equalTo(XContentType.SMILE));
9596
}
9697

@@ -128,15 +129,103 @@ public void testIncludes() throws Exception {
128129
)
129130
);
130131

132+
IndexableField sourceField = doc.rootDoc().getField("_source");
133+
Map<String, Object> sourceAsMap;
134+
try (XContentParser parser = createParser(JsonXContent.jsonXContent, new BytesArray(sourceField.binaryValue()))) {
135+
sourceAsMap = parser.map();
136+
}
137+
final IndexableField recoverySourceIndexableField = doc.rootDoc().getField("_recovery_source");
138+
assertNotNull(recoverySourceIndexableField);
139+
assertThat(sourceAsMap.containsKey("path1"), equalTo(true));
140+
assertThat(sourceAsMap.containsKey("path2"), equalTo(false));
141+
}
142+
143+
public void testIncludesForRecoverySource() throws Exception {
144+
String mapping = XContentFactory.jsonBuilder()
145+
.startObject()
146+
.startObject("type")
147+
.startObject("_source")
148+
.array("includes", new String[] { "path1*" })
149+
.array("recovery_source_includes", new String[]{"path2*"})
150+
.endObject()
151+
.endObject()
152+
.endObject()
153+
.toString();
154+
155+
DocumentMapper documentMapper = createIndex("test").mapperService()
156+
.documentMapperParser()
157+
.parse("type", new CompressedXContent(mapping));
158+
159+
ParsedDocument doc = documentMapper.parse(
160+
new SourceToParse(
161+
"test",
162+
"1",
163+
BytesReference.bytes(
164+
XContentFactory.jsonBuilder()
165+
.startObject()
166+
.startObject("path1")
167+
.field("field1", "value1")
168+
.endObject()
169+
.startObject("path2")
170+
.field("field2", "value2")
171+
.endObject()
172+
.endObject()
173+
),
174+
MediaTypeRegistry.JSON
175+
)
176+
);
177+
131178
IndexableField sourceField = doc.rootDoc().getField("_source");
132179
Map<String, Object> sourceAsMap;
133180
try (XContentParser parser = createParser(JsonXContent.jsonXContent, new BytesArray(sourceField.binaryValue()))) {
134181
sourceAsMap = parser.map();
135182
}
136183
assertThat(sourceAsMap.containsKey("path1"), equalTo(true));
137184
assertThat(sourceAsMap.containsKey("path2"), equalTo(false));
185+
186+
final IndexableField recoverySourceIndexableField = doc.rootDoc().getField("_recovery_source");
187+
assertNotNull(recoverySourceIndexableField);
188+
Map<String, Object> recoverySourceAsMap;
189+
try (XContentParser parser = createParser(JsonXContent.jsonXContent, new BytesArray(recoverySourceIndexableField.binaryValue()))) {
190+
recoverySourceAsMap = parser.map();
191+
}
192+
193+
assertThat(recoverySourceAsMap.containsKey("path1"), equalTo(false));
194+
assertThat(recoverySourceAsMap.containsKey("path2"), equalTo(true));
195+
}
196+
197+
public void testNoRecoverySourceAndNoSource_whenBothAreDisabled() throws Exception{
198+
String mapping = XContentFactory.jsonBuilder()
199+
.startObject()
200+
.startObject("type")
201+
.startObject("_source")
202+
.field("enabled", "false")
203+
.field("recovery_source_enabled", "false")
204+
.endObject()
205+
.endObject()
206+
.endObject()
207+
.toString();
208+
209+
DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser();
210+
DocumentMapper documentMapper = parser.parse("type", new CompressedXContent(mapping));
211+
BytesReference source = BytesReference.bytes(XContentFactory.jsonBuilder().startObject().field("field",
212+
"value").endObject());
213+
ParsedDocument doc = documentMapper.parse(
214+
new SourceToParse(
215+
"test",
216+
"1",
217+
source,
218+
MediaTypeRegistry.JSON
219+
)
220+
);
221+
222+
final IndexableField sourceIndexableField = doc.rootDoc().getField("_source");
223+
final IndexableField recoverySourceIndexableField = doc.rootDoc().getField("_recovery_source");
224+
assertNull(recoverySourceIndexableField);
225+
assertNull(sourceIndexableField);
138226
}
139227

228+
140229
public void testExcludes() throws Exception {
141230
String mapping = XContentFactory.jsonBuilder()
142231
.startObject()
@@ -171,13 +260,68 @@ public void testExcludes() throws Exception {
171260
)
172261
);
173262

263+
IndexableField sourceField = doc.rootDoc().getField("_source");
264+
Map<String, Object> sourceAsMap;
265+
try (XContentParser parser = createParser(JsonXContent.jsonXContent, new BytesArray(sourceField.binaryValue()))) {
266+
sourceAsMap = parser.map();
267+
}
268+
final IndexableField recoverySourceIndexableField = doc.rootDoc().getField("_recovery_source");
269+
assertNotNull(recoverySourceIndexableField);
270+
assertThat(sourceAsMap.containsKey("path1"), equalTo(false));
271+
assertThat(sourceAsMap.containsKey("path2"), equalTo(true));
272+
}
273+
274+
public void testExcludesForRecoverySource() throws Exception {
275+
String mapping = XContentFactory.jsonBuilder()
276+
.startObject()
277+
.startObject("type")
278+
.startObject("_source")
279+
.array("excludes", "path1*")
280+
.array("recovery_source_excludes", "path2*")
281+
.endObject()
282+
.endObject()
283+
.endObject()
284+
.toString();
285+
286+
DocumentMapper documentMapper = createIndex("test").mapperService()
287+
.documentMapperParser()
288+
.parse("type", new CompressedXContent(mapping));
289+
290+
ParsedDocument doc = documentMapper.parse(
291+
new SourceToParse(
292+
"test",
293+
"1",
294+
BytesReference.bytes(
295+
XContentFactory.jsonBuilder()
296+
.startObject()
297+
.startObject("path1")
298+
.field("field1", "value1")
299+
.endObject()
300+
.startObject("path2")
301+
.field("field2", "value2")
302+
.endObject()
303+
.endObject()
304+
),
305+
MediaTypeRegistry.JSON
306+
)
307+
);
308+
174309
IndexableField sourceField = doc.rootDoc().getField("_source");
175310
Map<String, Object> sourceAsMap;
176311
try (XContentParser parser = createParser(JsonXContent.jsonXContent, new BytesArray(sourceField.binaryValue()))) {
177312
sourceAsMap = parser.map();
178313
}
179314
assertThat(sourceAsMap.containsKey("path1"), equalTo(false));
180315
assertThat(sourceAsMap.containsKey("path2"), equalTo(true));
316+
317+
final IndexableField recoverySourceIndexableField = doc.rootDoc().getField("_recovery_source");
318+
assertNotNull(recoverySourceIndexableField);
319+
Map<String, Object> recoverySourceAsMap;
320+
try (XContentParser parser = createParser(JsonXContent.jsonXContent, new BytesArray(recoverySourceIndexableField.binaryValue()))) {
321+
recoverySourceAsMap = parser.map();
322+
}
323+
assertThat(recoverySourceAsMap.containsKey("path1"), equalTo(true));
324+
assertThat(recoverySourceAsMap.containsKey("path2"), equalTo(false));
181325
}
182326

183327
public void testEnabledNotUpdateable() throws Exception {

0 commit comments

Comments
 (0)