Skip to content

Commit f682fa6

Browse files
alecgrieserohadzeligerScottDugas
authored
Enable force continuations mode on all YamlIntegrationTests (#3228)
This updates the `YamlIntegrationTest`s so that they all run in "force continuations" mode. That means we get additional insight into how continuation handling works in those cases. This mostly accomplishes #3204. There are a few lingering queries that have not been moved over due to: #3206. In addition, several tests only run with force continuations on _newer_ versions. However, the majority of queries have at least some coverage of up-level and down-level testing for continuations to cover the issues identified as a part of this process. --------- Co-authored-by: ohad <ozeliger@apple.com> Co-authored-by: Scott Dugas <sdugas@apple.com> Co-authored-by: Scott Dugas <scott.dugas@gmail.com>
1 parent 1814f98 commit f682fa6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

48 files changed

+3344
-299
lines changed

yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/command/QueryExecutor.java

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,11 @@
2626
import com.apple.foundationdb.relational.api.RelationalResultSetMetaData;
2727
import com.apple.foundationdb.relational.api.exceptions.RelationalException;
2828
import com.apple.foundationdb.relational.api.metrics.RelationalMetric;
29+
import com.apple.foundationdb.relational.recordlayer.ContinuationImpl;
2930
import com.apple.foundationdb.relational.yamltests.AggregateResultSet;
3031
import com.apple.foundationdb.relational.yamltests.YamlConnection;
3132
import com.apple.foundationdb.relational.yamltests.command.parameterinjection.Parameter;
33+
import com.apple.foundationdb.relational.yamltests.server.SemanticVersion;
3234
import org.apache.logging.log4j.LogManager;
3335
import org.apache.logging.log4j.Logger;
3436
import org.junit.jupiter.api.Assertions;
@@ -53,6 +55,8 @@ public class QueryExecutor {
5355
private static final Logger logger = LogManager.getLogger(QueryExecutor.class);
5456
private static final int FORCED_MAX_ROWS = 1; // The maxRows number to use when we are forcing it on the test
5557
private static final int MAX_CONTINUATIONS_ALLOWED = 100;
58+
@SuppressWarnings("PMD.AvoidUsingHardCodedIP") // This is not an IP address
59+
private static final SemanticVersion STRICT_ASSERTIONS_CUTOFF = SemanticVersion.parse("4.1.9.0");
5660

5761
@Nonnull
5862
private final String query;
@@ -105,10 +109,9 @@ public Continuation execute(@Nonnull YamlConnection connection, @Nullable Contin
105109
if (continuation == null) {
106110
// no continuation - start the query execution from the beginning
107111
return executeQuery(connection, config, currentQuery, checkCache, maxRows);
108-
} else if (continuation.atBeginning()) {
112+
} else if (checkBeginningContinuation(continuation, connection)) {
109113
// Continuation cannot be at beginning if it was returned from a query
110-
reportTestFailure("Received continuation shouldn't be at beginning");
111-
return null;
114+
return ContinuationImpl.END;
112115
} else {
113116
// Have a continuation - continue
114117
return executeContinuation(connection, continuation, config, currentQuery, maxRows);
@@ -143,7 +146,7 @@ private Object executeStatementAndCheckCacheIfNeeded(@Nonnull Statement s, final
143146
final var toReturn = executeStatementAndCheckForceContinuations(s, statementHasQuery, queryString, connection, maxRows);
144147
final var postMetricCollector = connection.getMetricCollector();
145148
final var postValue = postMetricCollector.hasCounter(RelationalMetric.RelationalCount.PLAN_CACHE_TERTIARY_HIT) ?
146-
postMetricCollector.getCountsForCounter(RelationalMetric.RelationalCount.PLAN_CACHE_TERTIARY_HIT) : 0;
149+
postMetricCollector.getCountsForCounter(RelationalMetric.RelationalCount.PLAN_CACHE_TERTIARY_HIT) : 0;
147150
final var planFound = preMetricCollector != postMetricCollector ? postValue == 1 : postValue == preValue + 1;
148151
if (!planFound) {
149152
reportTestFailure("‼️ Expected to retrieve the plan from the cache at line " + lineNumber);
@@ -262,8 +265,9 @@ private Object executeStatementWithForcedContinuations(final @Nonnull Statement
262265
Continuation continuation = resultSet.getContinuation();
263266
int count = 0;
264267
while (!continuation.atEnd()) {
265-
if (continuation.atBeginning()) {
266-
reportTestFailure("Received continuation shouldn't be at beginning");
268+
if (checkBeginningContinuation(continuation, connection)) {
269+
continuation = ContinuationImpl.END;
270+
break;
267271
}
268272
try (var s2 = prepareContinuationStatement(connection, continuation, FORCED_MAX_ROWS)) {
269273
resultSet = (RelationalResultSet)executeStatement(s2, true, queryString);
@@ -273,7 +277,9 @@ private Object executeStatementWithForcedContinuations(final @Nonnull Statement
273277
results.add(resultSet);
274278
} else {
275279
// We assume that the last result is empty because of the maxRows:1
276-
Assertions.assertFalse(hasNext, "Result has more rows than maxRows allowed");
280+
if (STRICT_ASSERTIONS_CUTOFF.lesserVersions(connection.getVersions()).isEmpty()) {
281+
Assertions.assertFalse(hasNext, "End result should not have any associated value when maxRows is 1");
282+
}
277283
}
278284
}
279285
count += 1; // PMD failure for ++
@@ -290,6 +296,20 @@ private Object executeStatementWithForcedContinuations(final @Nonnull Statement
290296
}
291297
}
292298

299+
private boolean checkBeginningContinuation(Continuation continuation, YamlConnection connection) {
300+
if (continuation.atBeginning()) {
301+
if (STRICT_ASSERTIONS_CUTOFF.lesserVersions(connection.getVersions()).isEmpty()) {
302+
reportTestFailure("Received continuation shouldn't be at beginning");
303+
}
304+
if (logger.isInfoEnabled()) {
305+
logger.info("ignoring beginning continuation check for query '{}' at line {} (connection: {})",
306+
query, lineNumber, connection.getVersions());
307+
}
308+
return true;
309+
}
310+
return false;
311+
}
312+
293313
private static Object executeStatement(@Nonnull Statement s, final boolean statementHasQuery, @Nonnull String q) throws SQLException {
294314
final var execResult = statementHasQuery ? ((PreparedStatement) s).execute() : s.execute(q);
295315
return execResult ? s.getResultSet() : s.getUpdateCount();

yaml-tests/src/test/java/YamlIntegrationTests.java

Lines changed: 0 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
* limitations under the License.
1919
*/
2020

21-
import com.apple.foundationdb.relational.yamltests.ExcludeYamlTestConfig;
2221
import com.apple.foundationdb.relational.yamltests.MaintainYamlTestConfig;
2322
import com.apple.foundationdb.relational.yamltests.YamlTest;
2423
import com.apple.foundationdb.relational.yamltests.YamlTestConfigFilters;
@@ -39,43 +38,31 @@ public void showcasingTests(YamlTest.Runner runner) throws Exception {
3938
}
4039

4140
@TestTemplate
42-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
43-
reason = "Infinite continuation loop (https://github.com/FoundationDB/fdb-record-layer/issues/3095)")
4441
public void groupByTests(YamlTest.Runner runner) throws Exception {
4542
runner.runYamsql("groupby-tests.yamsql");
4643
}
4744

4845
@TestTemplate
49-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
50-
reason = "continuation verification (https://github.com/FoundationDB/fdb-record-layer/issues/3096)")
5146
public void standardTests(YamlTest.Runner runner) throws Exception {
5247
runner.runYamsql("standard-tests.yamsql");
5348
}
5449

5550
@TestTemplate
56-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
57-
reason = "continuation verification (https://github.com/FoundationDB/fdb-record-layer/issues/3096)")
5851
public void standardTestsWithProto(YamlTest.Runner runner) throws Exception {
5952
runner.runYamsql("standard-tests-proto.yamsql");
6053
}
6154

6255
@TestTemplate
63-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
64-
reason = "Continuation verification (https://github.com/FoundationDB/fdb-record-layer/issues/3096)")
6556
public void fieldIndexTestsProto(YamlTest.Runner runner) throws Exception {
6657
runner.runYamsql("field-index-tests-proto.yamsql");
6758
}
6859

6960
@TestTemplate
70-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
71-
reason = "Continuation verification (https://github.com/FoundationDB/fdb-record-layer/issues/3096)")
7261
public void standardTestsWithMetaData(YamlTest.Runner runner) throws Exception {
7362
runner.runYamsql("standard-tests-metadata.yamsql");
7463
}
7564

7665
@TestTemplate
77-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
78-
reason = "continuation verification (https://github.com/FoundationDB/fdb-record-layer/issues/3096)")
7966
public void nullOperator(YamlTest.Runner runner) throws Exception {
8067
runner.runYamsql("null-operator-tests.yamsql");
8168
}
@@ -102,8 +89,6 @@ public void joinTests(YamlTest.Runner runner) throws Exception {
10289
}
10390

10491
@TestTemplate
105-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
106-
reason = "Infinite continuation loop (https://github.com/FoundationDB/fdb-record-layer/issues/3095)")
10792
public void subqueryTests(YamlTest.Runner runner) throws Exception {
10893
runner.runYamsql("subquery-tests.yamsql");
10994
}
@@ -130,8 +115,6 @@ public void aggregateIndexTests(YamlTest.Runner runner) throws Exception {
130115
}
131116

132117
@TestTemplate
133-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
134-
reason = "Infinite continuation loop (https://github.com/FoundationDB/fdb-record-layer/issues/3095)")
135118
public void aggregateEmptyTable(YamlTest.Runner runner) throws Exception {
136119
runner.runYamsql("aggregate-empty-table.yamsql");
137120
}
@@ -192,8 +175,6 @@ void bytes(YamlTest.Runner runner) throws Exception {
192175
}
193176

194177
@TestTemplate
195-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
196-
reason = "Continuation verification (https://github.com/FoundationDB/fdb-record-layer/issues/3096)")
197178
void catalog(YamlTest.Runner runner) throws Exception {
198179
runner.runYamsql("catalog.yamsql");
199180
}
@@ -204,8 +185,6 @@ public void caseWhen(YamlTest.Runner runner) throws Exception {
204185
}
205186

206187
@TestTemplate
207-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
208-
reason = "maxRows ignored on update (https://github.com/FoundationDB/fdb-record-layer/issues/3100)")
209188
public void updateDeleteReturning(YamlTest.Runner runner) throws Exception {
210189
runner.runYamsql("update-delete-returning.yamsql");
211190
}
@@ -221,8 +200,6 @@ void functions(YamlTest.Runner runner) throws Exception {
221200
}
222201

223202
@TestTemplate
224-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
225-
reason = "Continuation verification (https://github.com/FoundationDB/fdb-record-layer/issues/3096)")
226203
void createDrop(YamlTest.Runner runner) throws Exception {
227204
runner.runYamsql("create-drop.yamsql");
228205
}
@@ -248,15 +225,11 @@ public void indexedFunctions(YamlTest.Runner runner) throws Exception {
248225
}
249226

250227
@TestTemplate
251-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
252-
reason = "Infinite continuation loop (https://github.com/FoundationDB/fdb-record-layer/issues/3095)")
253228
public void union(YamlTest.Runner runner) throws Exception {
254229
runner.runYamsql("union.yamsql");
255230
}
256231

257232
@TestTemplate
258-
@ExcludeYamlTestConfig(value = YamlTestConfigFilters.DO_NOT_FORCE_CONTINUATIONS,
259-
reason = "Infinite continuation loop (https://github.com/FoundationDB/fdb-record-layer/issues/3095)")
260233
public void unionEmptyTables(YamlTest.Runner runner) throws Exception {
261234
runner.runYamsql("union-empty-tables.yamsql");
262235
}

0 commit comments

Comments
 (0)