|
20 | 20 |
|
21 | 21 | import org.apache.flink.annotation.Internal;
|
22 | 22 | import org.apache.flink.api.connector.sink2.Sink;
|
| 23 | +import org.apache.flink.connector.base.sink.throwable.FatalExceptionClassifier; |
23 | 24 | import org.apache.flink.connector.base.sink.writer.AsyncSinkWriter;
|
24 | 25 | import org.apache.flink.connector.base.sink.writer.BufferedRequestState;
|
25 | 26 | import org.apache.flink.connector.base.sink.writer.ElementConverter;
|
|
34 | 35 | import org.apache.http.conn.ssl.TrustAllStrategy;
|
35 | 36 | import org.apache.http.impl.client.BasicCredentialsProvider;
|
36 | 37 | import org.apache.http.ssl.SSLContexts;
|
| 38 | +import org.opensearch.OpenSearchException; |
37 | 39 | import org.opensearch.action.ActionListener;
|
38 | 40 | import org.opensearch.action.bulk.BulkItemResponse;
|
39 | 41 | import org.opensearch.action.bulk.BulkRequest;
|
|
46 | 48 | import org.slf4j.LoggerFactory;
|
47 | 49 |
|
48 | 50 | import java.io.IOException;
|
| 51 | +import java.net.ConnectException; |
| 52 | +import java.net.NoRouteToHostException; |
49 | 53 | import java.security.KeyManagementException;
|
50 | 54 | import java.security.KeyStoreException;
|
51 | 55 | import java.security.NoSuchAlgorithmException;
|
@@ -73,6 +77,17 @@ class OpensearchAsyncWriter<InputT> extends AsyncSinkWriter<InputT, DocSerdeRequ
|
73 | 77 | private final Counter numRecordsOutErrorsCounter;
|
74 | 78 | private volatile boolean closed = false;
|
75 | 79 |
|
| 80 | + private static final FatalExceptionClassifier OPENSEARCH_FATAL_EXCEPTION_CLASSIFIER = |
| 81 | + FatalExceptionClassifier.createChain( |
| 82 | + new FatalExceptionClassifier( |
| 83 | + err -> |
| 84 | + err instanceof NoRouteToHostException |
| 85 | + || err instanceof ConnectException, |
| 86 | + err -> |
| 87 | + new OpenSearchException( |
| 88 | + "Could not connect to Opensearch cluster using provided hosts", |
| 89 | + err))); |
| 90 | + |
76 | 91 | /**
|
77 | 92 | * Constructor creating an Opensearch async writer.
|
78 | 93 | *
|
@@ -186,12 +201,31 @@ public void close() {
|
186 | 201 | }
|
187 | 202 | }
|
188 | 203 |
|
| 204 | + private boolean isRetryable(Throwable err) { |
| 205 | + // isFatal() is really isNotFatal() |
| 206 | + if (!OPENSEARCH_FATAL_EXCEPTION_CLASSIFIER.isFatal(err, getFatalExceptionCons())) { |
| 207 | + return false; |
| 208 | + } |
| 209 | + return true; |
| 210 | + } |
| 211 | + |
189 | 212 | private void handleFullyFailedBulkRequest(
|
190 | 213 | Throwable err,
|
191 | 214 | List<DocSerdeRequest<?>> requestEntries,
|
192 | 215 | Consumer<List<DocSerdeRequest<?>>> requestResult) {
|
| 216 | + final boolean retryable = isRetryable(err.getCause()); |
| 217 | + |
| 218 | + LOG.warn( |
| 219 | + "Opensearch AsyncWwriter failed to persist {} entries (retryable = {})", |
| 220 | + requestEntries.size(), |
| 221 | + retryable, |
| 222 | + err); |
| 223 | + |
193 | 224 | numRecordsOutErrorsCounter.inc(requestEntries.size());
|
194 |
| - requestResult.accept(requestEntries); |
| 225 | + |
| 226 | + if (retryable) { |
| 227 | + requestResult.accept(requestEntries); |
| 228 | + } |
195 | 229 | }
|
196 | 230 |
|
197 | 231 | private void handlePartiallyFailedBulkRequests(
|
|
0 commit comments