60
60
import org .opensearch .plugins .Plugin ;
61
61
import org .opensearch .test .OpenSearchIntegTestCase ;
62
62
import org .opensearch .test .ParameterizedStaticSettingsOpenSearchIntegTestCase ;
63
+ import org .hamcrest .MatcherAssert ;
63
64
64
65
import java .util .Arrays ;
65
66
import java .util .Collection ;
66
67
import java .util .Collections ;
67
68
import java .util .HashMap ;
68
69
import java .util .Map ;
70
+ import java .util .stream .Collectors ;
69
71
70
72
import static org .opensearch .common .xcontent .XContentFactory .jsonBuilder ;
71
73
import static org .opensearch .test .NodeRoles .nonIngestNode ;
74
+ import static org .hamcrest .Matchers .containsInAnyOrder ;
72
75
import static org .hamcrest .Matchers .equalTo ;
73
76
import static org .hamcrest .Matchers .instanceOf ;
74
77
import static org .hamcrest .Matchers .notNullValue ;
@@ -159,6 +162,14 @@ public void testSimulate() throws Exception {
159
162
}
160
163
161
164
public void testBulkWithIngestFailures () throws Exception {
165
+ runBulkTestWithRandomDocs (false );
166
+ }
167
+
168
+ public void testBulkWithIngestFailuresWithBatchSize () throws Exception {
169
+ runBulkTestWithRandomDocs (true );
170
+ }
171
+
172
+ private void runBulkTestWithRandomDocs (boolean shouldSetBatchSize ) throws Exception {
162
173
createIndex ("index" );
163
174
164
175
BytesReference source = BytesReference .bytes (
@@ -177,6 +188,9 @@ public void testBulkWithIngestFailures() throws Exception {
177
188
178
189
int numRequests = scaledRandomIntBetween (32 , 128 );
179
190
BulkRequest bulkRequest = new BulkRequest ();
191
+ if (shouldSetBatchSize ) {
192
+ bulkRequest .batchSize (numRequests );
193
+ }
180
194
for (int i = 0 ; i < numRequests ; i ++) {
181
195
IndexRequest indexRequest = new IndexRequest ("index" ).id (Integer .toString (i )).setPipeline ("_id" );
182
196
indexRequest .source (Requests .INDEX_CONTENT_TYPE , "field" , "value" , "fail" , i % 2 == 0 );
@@ -209,6 +223,103 @@ public void testBulkWithIngestFailures() throws Exception {
209
223
assertTrue (deletePipelineResponse .isAcknowledged ());
210
224
}
211
225
226
+ public void testBulkWithIngestFailuresBatch () throws Exception {
227
+ createIndex ("index" );
228
+
229
+ BytesReference source = BytesReference .bytes (
230
+ jsonBuilder ().startObject ()
231
+ .field ("description" , "my_pipeline" )
232
+ .startArray ("processors" )
233
+ .startObject ()
234
+ .startObject ("test" )
235
+ .endObject ()
236
+ .endObject ()
237
+ .endArray ()
238
+ .endObject ()
239
+ );
240
+ PutPipelineRequest putPipelineRequest = new PutPipelineRequest ("_id" , source , MediaTypeRegistry .JSON );
241
+ client ().admin ().cluster ().putPipeline (putPipelineRequest ).get ();
242
+
243
+ BulkRequest bulkRequest = new BulkRequest ();
244
+ bulkRequest .batchSize (2 );
245
+ bulkRequest .add (
246
+ new IndexRequest ("index" ).id ("_fail" ).setPipeline ("_id" ).source (Requests .INDEX_CONTENT_TYPE , "field" , "value" , "fail" , true )
247
+ );
248
+ bulkRequest .add (
249
+ new IndexRequest ("index" ).id ("_success" ).setPipeline ("_id" ).source (Requests .INDEX_CONTENT_TYPE , "field" , "value" , "fail" , false )
250
+ );
251
+
252
+ BulkResponse response = client ().bulk (bulkRequest ).actionGet ();
253
+ MatcherAssert .assertThat (response .getItems ().length , equalTo (bulkRequest .requests ().size ()));
254
+
255
+ Map <String , BulkItemResponse > results = Arrays .stream (response .getItems ())
256
+ .collect (Collectors .toMap (BulkItemResponse ::getId , r -> r ));
257
+
258
+ MatcherAssert .assertThat (results .keySet (), containsInAnyOrder ("_fail" , "_success" ));
259
+ assertNotNull (results .get ("_fail" ).getFailure ());
260
+ assertNull (results .get ("_success" ).getFailure ());
261
+
262
+ // verify field of successful doc
263
+ Map <String , Object > successDoc = client ().prepareGet ("index" , "_success" ).get ().getSourceAsMap ();
264
+ assertThat (successDoc .get ("processed" ), equalTo (true ));
265
+
266
+ // cleanup
267
+ AcknowledgedResponse deletePipelineResponse = client ().admin ().cluster ().prepareDeletePipeline ("_id" ).get ();
268
+ assertTrue (deletePipelineResponse .isAcknowledged ());
269
+ }
270
+
271
+ public void testBulkWithIngestFailuresAndDropBatch () throws Exception {
272
+ createIndex ("index" );
273
+
274
+ BytesReference source = BytesReference .bytes (
275
+ jsonBuilder ().startObject ()
276
+ .field ("description" , "my_pipeline" )
277
+ .startArray ("processors" )
278
+ .startObject ()
279
+ .startObject ("test" )
280
+ .endObject ()
281
+ .endObject ()
282
+ .endArray ()
283
+ .endObject ()
284
+ );
285
+ PutPipelineRequest putPipelineRequest = new PutPipelineRequest ("_id" , source , MediaTypeRegistry .JSON );
286
+ client ().admin ().cluster ().putPipeline (putPipelineRequest ).get ();
287
+
288
+ BulkRequest bulkRequest = new BulkRequest ();
289
+ bulkRequest .batchSize (3 );
290
+ bulkRequest .add (
291
+ new IndexRequest ("index" ).id ("_fail" ).setPipeline ("_id" ).source (Requests .INDEX_CONTENT_TYPE , "field" , "value" , "fail" , true )
292
+ );
293
+ bulkRequest .add (
294
+ new IndexRequest ("index" ).id ("_success" ).setPipeline ("_id" ).source (Requests .INDEX_CONTENT_TYPE , "field" , "value" , "fail" , false )
295
+ );
296
+ bulkRequest .add (
297
+ new IndexRequest ("index" ).id ("_drop" ).setPipeline ("_id" ).source (Requests .INDEX_CONTENT_TYPE , "field" , "value" , "drop" , true )
298
+ );
299
+
300
+ BulkResponse response = client ().bulk (bulkRequest ).actionGet ();
301
+ MatcherAssert .assertThat (response .getItems ().length , equalTo (bulkRequest .requests ().size ()));
302
+
303
+ Map <String , BulkItemResponse > results = Arrays .stream (response .getItems ())
304
+ .collect (Collectors .toMap (BulkItemResponse ::getId , r -> r ));
305
+
306
+ MatcherAssert .assertThat (results .keySet (), containsInAnyOrder ("_fail" , "_success" , "_drop" ));
307
+ assertNotNull (results .get ("_fail" ).getFailure ());
308
+ assertNull (results .get ("_success" ).getFailure ());
309
+ assertNull (results .get ("_drop" ).getFailure ());
310
+
311
+ // verify dropped doc not in index
312
+ assertNull (client ().prepareGet ("index" , "_drop" ).get ().getSourceAsMap ());
313
+
314
+ // verify field of successful doc
315
+ Map <String , Object > successDoc = client ().prepareGet ("index" , "_success" ).get ().getSourceAsMap ();
316
+ assertThat (successDoc .get ("processed" ), equalTo (true ));
317
+
318
+ // cleanup
319
+ AcknowledgedResponse deletePipelineResponse = client ().admin ().cluster ().prepareDeletePipeline ("_id" ).get ();
320
+ assertTrue (deletePipelineResponse .isAcknowledged ());
321
+ }
322
+
212
323
public void testBulkWithUpsert () throws Exception {
213
324
createIndex ("index" );
214
325
0 commit comments