From 387a648cfa8d122f5012a69c862556ad4db9bfe8 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 21 May 2025 10:57:24 -0700 Subject: [PATCH 01/15] save --- .../provider/foundationdb/KeyValueCursorBase.java | 10 ++++++++++ .../src/main/proto/record_cursor.proto | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java index ec5f7b9d17..4efed46991 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java @@ -33,6 +33,7 @@ import com.apple.foundationdb.record.KeyRange; import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.RecordCursorContinuation; +import com.apple.foundationdb.record.RecordCursorProto; import com.apple.foundationdb.record.RecordCursorResult; import com.apple.foundationdb.record.ScanProperties; import com.apple.foundationdb.record.TupleRange; @@ -46,6 +47,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.awt.RenderingHints; import java.util.Arrays; import java.util.Optional; import java.util.concurrent.CompletableFuture; @@ -171,6 +173,14 @@ public byte[] toBytes() { } return Arrays.copyOfRange(lastKey, prefixLength, lastKey.length); } + + @Nonnull + private RecordCursorProto.KeyValueCursorContinuation toProto() { + return RecordCursorProto.KeyValueCursorContinuation.newBuilder() + .setLastKey(ByteString.copyFrom(lastKey)) + .setPrefixLength(prefixLength) + .build(); + } } /** diff --git a/fdb-record-layer-core/src/main/proto/record_cursor.proto b/fdb-record-layer-core/src/main/proto/record_cursor.proto index 5e9adb2858..a191525ce7 100644 --- a/fdb-record-layer-core/src/main/proto/record_cursor.proto +++ b/fdb-record-layer-core/src/main/proto/record_cursor.proto @@ -140,4 +140,9 @@ message RecursiveCursorContinuation { message RangeCursorContinuation { optional int64 nextPosition = 1; +} + +message KeyValueCursorContinuation { + optional bytes lastKey = 1; + optional int32 prefixLength = 2; } \ No newline at end of file From 0fd847e61b51b69aff848d8183ccfb8cbd1b6f07 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 22 May 2025 16:03:12 -0700 Subject: [PATCH 02/15] save --- .../IndexPrefetchRangeKeyValueCursor.java | 7 +- .../provider/foundationdb/KeyValueCursor.java | 7 +- .../foundationdb/KeyValueCursorBase.java | 73 +++++++-- .../src/main/proto/record_cursor.proto | 4 +- .../foundationdb/KeyValueCursorTest.java | 151 +++++++++++++++--- 5 files changed, 193 insertions(+), 49 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexPrefetchRangeKeyValueCursor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexPrefetchRangeKeyValueCursor.java index df4ad626ec..71bda251a3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexPrefetchRangeKeyValueCursor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/IndexPrefetchRangeKeyValueCursor.java @@ -41,9 +41,10 @@ private IndexPrefetchRangeKeyValueCursor(@Nonnull final FDBRecordContext context @Nonnull final AsyncIterator iterator, int prefixLength, @Nonnull final CursorLimitManager limitManager, - int valuesLimit) { + int valuesLimit, + SerializationMode serializationMode) { - super(context, iterator, prefixLength, limitManager, valuesLimit); + super(context, iterator, prefixLength, limitManager, valuesLimit, serializationMode); } /** @@ -69,7 +70,7 @@ public IndexPrefetchRangeKeyValueCursor build() { AsyncIterator iterator = getTransaction() .getMappedRange(getBegin(), getEnd(), mapper, getLimit(), isReverse(), getStreamingMode()) .iterator(); - return new IndexPrefetchRangeKeyValueCursor(getContext(), iterator, getPrefixLength(), getLimitManager(), getValuesLimit()); + return new IndexPrefetchRangeKeyValueCursor(getContext(), iterator, getPrefixLength(), getLimitManager(), getValuesLimit(), serializationMode); } @Override diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursor.java index 6534ceb5a4..ed5ea7e3b5 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursor.java @@ -38,8 +38,9 @@ private KeyValueCursor(@Nonnull final FDBRecordContext context, @Nonnull final AsyncIterator iterator, int prefixLength, @Nonnull final CursorLimitManager limitManager, - int valuesLimit) { - super(context, iterator, prefixLength, limitManager, valuesLimit); + int valuesLimit, + SerializationMode serializationMode) { + super(context, iterator, prefixLength, limitManager, valuesLimit, serializationMode); } /** @@ -77,7 +78,7 @@ public KeyValueCursor build() { final AsyncIterator iterator = getTransaction() .getRange(getBegin(), getEnd(), getLimit(), isReverse(), getStreamingMode()) .iterator(); - return new KeyValueCursor(getContext(), iterator, getPrefixLength(), getLimitManager(), getValuesLimit()); + return new KeyValueCursor(getContext(), iterator, getPrefixLength(), getLimitManager(), getValuesLimit(), serializationMode); } @Override diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java index 4efed46991..475126aa01 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java @@ -41,8 +41,10 @@ import com.apple.foundationdb.record.cursors.BaseCursor; import com.apple.foundationdb.record.cursors.CursorLimitManager; import com.apple.foundationdb.subspace.Subspace; +import com.apple.foundationdb.tuple.ByteArrayUtil2; import com.apple.foundationdb.tuple.Tuple; import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.ZeroCopyByteString; import javax.annotation.Nonnull; @@ -67,18 +69,22 @@ public abstract class KeyValueCursorBase extends AsyncIterat // the pointer may be mutated, but the actual array must never be mutated or continuations will break @Nullable private byte[] lastKey; + @Nonnull + private SerializationMode serializationMode; protected KeyValueCursorBase(@Nonnull final FDBRecordContext context, @Nonnull final AsyncIterator iterator, int prefixLength, @Nonnull final CursorLimitManager limitManager, - int valuesLimit) { + int valuesLimit, + SerializationMode serializationMode) { super(context.getExecutor(), iterator); this.context = context; this.prefixLength = prefixLength; this.limitManager = limitManager; this.valuesLimit = valuesLimit; + this.serializationMode = serializationMode; context.instrument(FDBStoreTimer.DetailEvents.GET_SCAN_RANGE_RAW_FIRST_CHUNK, iterator.onHasNext()); } @@ -133,21 +139,23 @@ public RecordCursorResult getNext() { @Nonnull private RecordCursorContinuation continuationHelper() { - return new Continuation(lastKey, prefixLength); + return new Continuation(lastKey, prefixLength, serializationMode); } private static class Continuation implements RecordCursorContinuation { @Nullable private final byte[] lastKey; private final int prefixLength; + private final SerializationMode serializationMode; - public Continuation(@Nullable final byte[] lastKey, final int prefixLength) { + public Continuation(@Nullable final byte[] lastKey, final int prefixLength, final SerializationMode serializationMode) { // Note that doing this without a full copy is dangerous if the array is ever mutated. // Currently, this never happens and the only thing that changes is which array lastKey points to. // However, if logic in KeyValueCursor or KeyValue changes, this could break continuations. // To resolve it, we could resort to doing a full copy here, although that's somewhat expensive. this.lastKey = lastKey; this.prefixLength = prefixLength; + this.serializationMode = serializationMode; } @Override @@ -158,31 +166,46 @@ public boolean isEnd() { @Nonnull @Override public ByteString toByteString() { - if (lastKey == null) { - return ByteString.EMPTY; + if (serializationMode == SerializationMode.TO_OLD) { + if (lastKey == null) { + return ByteString.EMPTY; + } + ByteString base = ZeroCopyByteString.wrap(lastKey); + return base.substring(prefixLength, lastKey.length); + } else { + return toProto().toByteString(); } - ByteString base = ZeroCopyByteString.wrap(lastKey); - return base.substring(prefixLength, lastKey.length); } @Nullable @Override public byte[] toBytes() { - if (lastKey == null) { - return null; + if (serializationMode == SerializationMode.TO_OLD) { + if (lastKey == null) { + return null; + } + return Arrays.copyOfRange(lastKey, prefixLength, lastKey.length); + } else { + return toProto().toByteArray(); } - return Arrays.copyOfRange(lastKey, prefixLength, lastKey.length); } @Nonnull private RecordCursorProto.KeyValueCursorContinuation toProto() { - return RecordCursorProto.KeyValueCursorContinuation.newBuilder() - .setLastKey(ByteString.copyFrom(lastKey)) - .setPrefixLength(prefixLength) - .build(); + RecordCursorProto.KeyValueCursorContinuation.Builder builder = RecordCursorProto.KeyValueCursorContinuation.newBuilder(); + if (lastKey != null) { + ByteString base = ZeroCopyByteString.wrap(lastKey); + builder.setContinuation(base.substring(prefixLength, lastKey.length)); + } + return builder.setIsStart(false).build(); } } + public enum SerializationMode { + TO_OLD, + TO_NEW + } + /** * A builder for {@link KeyValueCursorBase}. * @param the type of the concrete subclass of the builder @@ -218,6 +241,8 @@ public abstract static class Builder> { private StreamingMode streamingMode; private KeySelector begin; private KeySelector end; + // default to be old now + protected SerializationMode serializationMode = SerializationMode.TO_OLD; protected Builder(@Nonnull Subspace subspace) { this.subspace = subspace; @@ -258,9 +283,20 @@ protected void prepare() { reverse = scanProperties.isReverse(); if (continuation != null) { - final byte[] continuationBytes = new byte[prefixLength + continuation.length]; + byte[] realContinuation; + if (serializationMode == SerializationMode.TO_OLD) { + realContinuation = continuation; + } else { + try { + RecordCursorProto.KeyValueCursorContinuation keyValueCursorContinuation = RecordCursorProto.KeyValueCursorContinuation.parseFrom(continuation); + realContinuation = keyValueCursorContinuation.getContinuation().toByteArray(); + } catch (InvalidProtocolBufferException ex) { + realContinuation = continuation; + } + } + final byte[] continuationBytes = new byte[prefixLength + realContinuation.length]; System.arraycopy(lowBytes, 0, continuationBytes, 0, prefixLength); - System.arraycopy(continuation, 0, continuationBytes, prefixLength, continuation.length); + System.arraycopy(realContinuation, 0, continuationBytes, prefixLength, realContinuation.length); if (reverse) { highBytes = continuationBytes; highEndpoint = EndpointType.CONTINUATION; @@ -344,6 +380,11 @@ public T setHigh(@Nonnull byte[] highBytes, @Nonnull EndpointType highEndpoint) return self(); } + public T setSerializationMode(@Nonnull final SerializationMode serializationMode) { + this.serializationMode = serializationMode; + return self(); + } + /** * Calculate the key prefix length for the returned values. This will be used to derive the primary key used in * the calculated continuation. diff --git a/fdb-record-layer-core/src/main/proto/record_cursor.proto b/fdb-record-layer-core/src/main/proto/record_cursor.proto index a191525ce7..77e0fa4946 100644 --- a/fdb-record-layer-core/src/main/proto/record_cursor.proto +++ b/fdb-record-layer-core/src/main/proto/record_cursor.proto @@ -143,6 +143,6 @@ message RangeCursorContinuation { } message KeyValueCursorContinuation { - optional bytes lastKey = 1; - optional int32 prefixLength = 2; + optional bytes continuation = 1; + optional bool isStart = 2; } \ No newline at end of file diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java index d97e4b0ef9..d426804c61 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java @@ -44,6 +44,8 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import java.util.Arrays; import java.util.Collections; @@ -90,14 +92,56 @@ public void runBefore() { }); } + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void all(KeyValueCursorBase.SerializationMode serializationMode) { + fdb.run(context -> { + KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) + .setContext(context) + .setRange(TupleRange.ALL) + .setContinuation(null) + .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) + .build(); + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 5; j++) { + KeyValue kv = cursor.getNext().get(); + assertArrayEquals(subspace.pack(Tuple.from(i, j)), kv.getKey()); + assertArrayEquals(Tuple.from(i, j).pack(), kv.getValue()); + } + } + assertThat(cursor.getNext().hasNext(), is(false)); + + cursor = KeyValueCursor.Builder.withSubspace(subspace) + .setContext(context) + .setRange(TupleRange.ALL) + .setContinuation(null) + .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(10).build())) + .setSerializationMode(serializationMode) + .build(); + assertEquals(10, (int)cursor.getCount().join()); + cursor = KeyValueCursor.Builder.withSubspace(subspace) + .setContext(context) + .setRange(TupleRange.ALL) + .setContinuation(cursor.getNext().getContinuation().toBytes()) + .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) + .build(); + assertEquals(15, (int)cursor.getCount().join()); + + return null; + }); + } + @Test - public void all() { + public void allOldThanNew() { fdb.run(context -> { KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setRange(TupleRange.ALL) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(KeyValueCursorBase.SerializationMode.TO_OLD) .build(); for (int i = 0; i < 5; i++) { for (int j = 0; j < 5; j++) { @@ -113,6 +157,7 @@ public void all() { .setRange(TupleRange.ALL) .setContinuation(null) .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(10).build())) + .setSerializationMode(KeyValueCursorBase.SerializationMode.TO_OLD) .build(); assertEquals(10, (int)cursor.getCount().join()); cursor = KeyValueCursor.Builder.withSubspace(subspace) @@ -120,6 +165,7 @@ public void all() { .setRange(TupleRange.ALL) .setContinuation(cursor.getNext().getContinuation().toBytes()) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(KeyValueCursorBase.SerializationMode.TO_NEW) .build(); assertEquals(15, (int)cursor.getCount().join()); @@ -128,13 +174,39 @@ public void all() { } @Test - public void beginsWith() { + public void lastRow() { + fdb.run(context -> { + KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) + .setContext(context) + .setRange(TupleRange.ALL) + .setContinuation(null) + .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(25).build())) + .setSerializationMode(KeyValueCursorBase.SerializationMode.TO_OLD) + .build(); + assertEquals(25, (int)cursor.getCount().join()); + cursor = KeyValueCursor.Builder.withSubspace(subspace) + .setContext(context) + .setRange(TupleRange.ALL) + .setContinuation(cursor.getNext().getContinuation().toBytes()) + .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(KeyValueCursorBase.SerializationMode.TO_NEW) + .build(); + assertEquals(0, (int)cursor.getCount().join()); + + return null; + }); + } + + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void beginsWith(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setRange(TupleRange.allOf(Tuple.from(3))) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); for (int j = 0; j < 5; j++) { KeyValue kv = cursor.getNext().get(); @@ -148,6 +220,7 @@ public void beginsWith() { .setRange(TupleRange.allOf(Tuple.from(3))) .setContinuation(null) .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(2).build())) + .setSerializationMode(serializationMode) .build(); assertEquals(2, (int)cursor.getCount().join()); cursor = KeyValueCursor.Builder.withSubspace(subspace) @@ -155,6 +228,7 @@ public void beginsWith() { .setRange(TupleRange.allOf(Tuple.from(3))) .setContinuation(cursor.getNext().getContinuation().toBytes()) .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(3).build())) + .setSerializationMode(serializationMode) .build(); assertEquals(3, (int)cursor.getCount().join()); @@ -162,8 +236,9 @@ public void beginsWith() { }); } - @Test - public void inclusiveRange() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void inclusiveRange(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) @@ -171,6 +246,7 @@ public void inclusiveRange() { .setHigh(Tuple.from(4, 2), EndpointType.RANGE_INCLUSIVE) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); assertEquals(Arrays.asList(Tuple.from(3L, 3L), Tuple.from(3L, 4L), Tuple.from(4L, 0L), Tuple.from(4L, 1L), Tuple.from(4L, 2L)), cursor.map(KeyValue::getValue).map(Tuple::fromBytes).asList().join()); @@ -181,6 +257,7 @@ public void inclusiveRange() { .setHigh(Tuple.from(4, 2), EndpointType.RANGE_INCLUSIVE) .setContinuation(null) .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(2).build())) + .setSerializationMode(serializationMode) .build(); assertEquals(Arrays.asList(Tuple.from(3L, 3L), Tuple.from(3L, 4L)), cursor.map(KeyValue::getValue).map(Tuple::fromBytes).asList().join()); @@ -190,6 +267,7 @@ public void inclusiveRange() { .setHigh(Tuple.from(4, 2), EndpointType.RANGE_INCLUSIVE) .setContinuation(cursor.getNext().getContinuation().toBytes()) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); assertEquals(Arrays.asList(Tuple.from(4L, 0L), Tuple.from(4L, 1L), Tuple.from(4L, 2L)), cursor.map(KeyValue::getValue).map(Tuple::fromBytes).asList().join()); @@ -198,8 +276,9 @@ public void inclusiveRange() { }); } - @Test - public void exclusiveRange() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void exclusiveRange(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) @@ -207,6 +286,7 @@ public void exclusiveRange() { .setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); assertEquals(Arrays.asList(Tuple.from(3L, 4L), Tuple.from(4L, 0L), Tuple.from(4L, 1L)), cursor.map(KeyValue::getValue).map(Tuple::fromBytes).asList().join()); @@ -217,6 +297,7 @@ public void exclusiveRange() { .setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE) .setContinuation(null) .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(2).build())) + .setSerializationMode(serializationMode) .build(); assertEquals(Arrays.asList(Tuple.from(3L, 4L), Tuple.from(4L, 0L)), cursor.map(KeyValue::getValue).map(Tuple::fromBytes).asList().join()); @@ -226,6 +307,7 @@ public void exclusiveRange() { .setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE) .setContinuation(cursor.getNext().getContinuation().toBytes()) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); assertEquals(Collections.singletonList(Tuple.from(4L, 1L)), cursor.map(KeyValue::getValue).map(Tuple::fromBytes).asList().join()); @@ -234,8 +316,9 @@ public void exclusiveRange() { }); } - @Test - public void inclusiveNull() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void inclusiveNull(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordCursorIterator cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) @@ -243,6 +326,7 @@ public void inclusiveNull() { .setHigh((Tuple) null, EndpointType.RANGE_INCLUSIVE) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build() .asIterator(); for (int j = 0; j < 5; j++) { @@ -256,8 +340,9 @@ public void inclusiveNull() { }); } - @Test - public void exclusiveNull() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void exclusiveNull(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordCursorIterator cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) @@ -265,6 +350,7 @@ public void exclusiveNull() { .setHigh((Tuple) null, EndpointType.RANGE_EXCLUSIVE) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build() .asIterator(); assertThat(cursor.hasNext(), is(false)); @@ -273,13 +359,15 @@ public void exclusiveNull() { }); } - @Test - public void noNextReasons() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void noNextReasons(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setRange(TupleRange.allOf(Tuple.from(3))) .setContinuation(null) + .setSerializationMode(serializationMode) .setScanProperties(ScanProperties.FORWARD_SCAN.with(props -> props.setReturnedRowLimit(3))) .build(); assertEquals(Arrays.asList(Tuple.from(3L, 0L), Tuple.from(3L, 1L), Tuple.from(3L, 2L)), @@ -290,6 +378,7 @@ public void noNextReasons() { .setContext(context) .setRange(TupleRange.allOf(Tuple.from(3))) .setContinuation(result.getContinuation().toBytes()) + .setSerializationMode(serializationMode) .setScanProperties(ScanProperties.FORWARD_SCAN.with(props -> props.setReturnedRowLimit(3))) .build(); assertEquals(Arrays.asList(Tuple.from(3L, 3L), Tuple.from(3L, 4L)), @@ -306,14 +395,16 @@ private ScanProperties forwardScanWithLimiter(RecordScanLimiter limiter) { return new ScanProperties(ExecuteProperties.SERIAL_EXECUTE.setState(new ExecuteState(limiter, null))); } - @Test - public void simpleScanLimit() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void simpleScanLimit(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordScanLimiter limiter = RecordScanLimiterFactory.enforce(2); KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setRange(TupleRange.ALL) .setScanProperties(forwardScanWithLimiter(limiter)) + .setSerializationMode(serializationMode) .build(); assertEquals(2, (int) cursor.getCount().join()); RecordCursorResult result = cursor.getNext(); @@ -324,8 +415,9 @@ public void simpleScanLimit() { }); } - @Test - public void limitNotReached() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void limitNotReached(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordScanLimiter limiter = RecordScanLimiterFactory.enforce(4); KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) @@ -333,6 +425,7 @@ public void limitNotReached() { .setLow(Tuple.from(3, 3), EndpointType.RANGE_EXCLUSIVE) .setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE) .setScanProperties(forwardScanWithLimiter(limiter)) + .setSerializationMode(serializationMode) .build(); assertEquals(3, (int) cursor.getCount().join()); RecordCursorResult result = cursor.getNext(); @@ -347,15 +440,17 @@ private boolean hasNextAndAdvance(KeyValueCursor cursor) { return cursor.getNext().hasNext(); } - @Test - public void sharedLimiter() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void sharedLimiter(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordScanLimiter limiter = RecordScanLimiterFactory.enforce(4); KeyValueCursor.Builder builder = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setLow(Tuple.from(3, 3), EndpointType.RANGE_EXCLUSIVE) .setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE) - .setScanProperties(forwardScanWithLimiter(limiter)); + .setScanProperties(forwardScanWithLimiter(limiter)) + .setSerializationMode(serializationMode); KeyValueCursor cursor1 = builder.build(); KeyValueCursor cursor2 = builder.build(); @@ -374,8 +469,9 @@ public void sharedLimiter() { }); } - @Test - public void limiterWithLookahead() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void limiterWithLookahead(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordScanLimiter limiter = RecordScanLimiterFactory.enforce(1); KeyValueCursor kvCursor = KeyValueCursor.Builder.withSubspace(subspace) @@ -383,6 +479,7 @@ public void limiterWithLookahead() { .setLow(Tuple.from(3, 3), EndpointType.RANGE_EXCLUSIVE) .setHigh(Tuple.from(4, 2), EndpointType.RANGE_EXCLUSIVE) .setScanProperties(forwardScanWithLimiter(limiter)) + .setSerializationMode(serializationMode) .build(); RecordCursor cursor = kvCursor.skip(2); // should exhaust limit first RecordCursorResult result = cursor.getNext(); @@ -393,14 +490,16 @@ public void limiterWithLookahead() { }); } - @Test - public void emptyScan() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void emptyScan(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setRange(TupleRange.allOf(Tuple.from(9))) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); RecordCursorResult result = cursor.getNext(); assertFalse(result.hasNext()); @@ -411,14 +510,16 @@ public void emptyScan() { }); } - @Test - public void emptyScanSplit() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void emptyScanSplit(KeyValueCursorBase.SerializationMode serializationMode) { fdb.run(context -> { RecordCursor kvCursor = KeyValueCursor.Builder.withSubspace(subspace) .setContext(context) .setRange(TupleRange.allOf(Tuple.from(9))) .setContinuation(null) .setScanProperties(ScanProperties.FORWARD_SCAN) + .setSerializationMode(serializationMode) .build(); RecordCursor cursor = new SplitHelper.KeyValueUnsplitter(context, subspace, kvCursor, false, null, false, new CursorLimitManager(context, ScanProperties.FORWARD_SCAN)); From 3ce0259c0aa82600fbce87774ccc7b9d105e433e Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Tue, 27 May 2025 20:13:45 -0700 Subject: [PATCH 03/15] save --- .../foundationdb/KeyValueCursorBase.java | 12 +++++++----- .../src/main/proto/record_cursor.proto | 2 +- .../resources/aggregate-index-tests-count.yamsql | 12 ++++++++---- .../test/resources/aggregate-index-tests.yamsql | 16 ++++++++++++---- 4 files changed, 28 insertions(+), 14 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java index 475126aa01..429703b365 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java @@ -70,14 +70,14 @@ public abstract class KeyValueCursorBase extends AsyncIterat @Nullable private byte[] lastKey; @Nonnull - private SerializationMode serializationMode; + private final SerializationMode serializationMode; protected KeyValueCursorBase(@Nonnull final FDBRecordContext context, @Nonnull final AsyncIterator iterator, int prefixLength, @Nonnull final CursorLimitManager limitManager, int valuesLimit, - SerializationMode serializationMode) { + @Nonnull SerializationMode serializationMode) { super(context.getExecutor(), iterator); this.context = context; @@ -193,11 +193,13 @@ public byte[] toBytes() { @Nonnull private RecordCursorProto.KeyValueCursorContinuation toProto() { RecordCursorProto.KeyValueCursorContinuation.Builder builder = RecordCursorProto.KeyValueCursorContinuation.newBuilder(); - if (lastKey != null) { + if (lastKey == null) { + builder.setIsEnd(true); + } else { ByteString base = ZeroCopyByteString.wrap(lastKey); - builder.setContinuation(base.substring(prefixLength, lastKey.length)); + builder.setContinuation(base.substring(prefixLength, lastKey.length)).setIsEnd(false); } - return builder.setIsStart(false).build(); + return builder.build(); } } diff --git a/fdb-record-layer-core/src/main/proto/record_cursor.proto b/fdb-record-layer-core/src/main/proto/record_cursor.proto index 77e0fa4946..7b06c76a6a 100644 --- a/fdb-record-layer-core/src/main/proto/record_cursor.proto +++ b/fdb-record-layer-core/src/main/proto/record_cursor.proto @@ -144,5 +144,5 @@ message RangeCursorContinuation { message KeyValueCursorContinuation { optional bytes continuation = 1; - optional bool isStart = 2; + optional bool isEnd = 2; } \ No newline at end of file diff --git a/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql b/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql index c04499e1ce..3f31b84936 100644 --- a/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql +++ b/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql @@ -48,8 +48,9 @@ test_block: - query: select count(*) from t1 - explain: "AISCAN(MV1 <,> BY_GROUP -> [_0: VALUE:[0]]) | MAP (_ AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)" # Cannot run with FORCE_CONTINUATIONS due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 0 + - maxRows: 1 - result: [{4}] + - result: [] - - query: select count(*) from t1 group by col2 - explain: "AISCAN(MV2 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)" @@ -58,8 +59,9 @@ test_block: - query: select count(col1) from t1 - explain: "AISCAN(MV3 <,> BY_GROUP -> [_0: VALUE:[0]]) | MAP (_ AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)" # Cannot run with FORCE_CONTINUATIONS due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 0 + - maxRows: 1 - result: [{2}] + - result: [] - - query: select count(col1) from t1 group by col2 - explain: "AISCAN(MV4 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)" @@ -86,8 +88,9 @@ test_block: - query: select count(*) from t2 - explain: "ISCAN(MV5 <,>) | MAP (_ AS _0) | AGG (count_star(*) AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)" # Cannot run with FORCE_CONTINUATIONS due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 0 + - maxRows: 1 - result: [{4}] + - result: [] - - query: select count(*) from t2 group by col2 # Plan deserialization previously failed : https://github.com/FoundationDB/fdb-record-layer/issues/3214 @@ -109,8 +112,9 @@ test_block: - query: select count(col1) from t2 - explain: "ISCAN(MV5 <,>) | MAP (_ AS _0) | AGG (count(_._0.COL1) AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)" # Cannot run with FORCE_CONTINUATIONS due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 0 + - maxRows: 1 - result: [{2}] + - result: [] - - query: select count(col1) from t2 group by col2 # Plan deserialization previously failed : https://github.com/FoundationDB/fdb-record-layer/issues/3214 diff --git a/yaml-tests/src/test/resources/aggregate-index-tests.yamsql b/yaml-tests/src/test/resources/aggregate-index-tests.yamsql index c886c92ceb..1569d3bb94 100644 --- a/yaml-tests/src/test/resources/aggregate-index-tests.yamsql +++ b/yaml-tests/src/test/resources/aggregate-index-tests.yamsql @@ -257,8 +257,12 @@ test_block: # Cannot use FORCE_CONTINUATIONS with older versions due to: https://github.com/FoundationDB/fdb-record-layer/issues/3096 # (Extra values being produced after exhausting source of an aggregate cursor) # Can remove once we do not care about backwards compatibility before 4.1.9.0 - - maxRows: 0 - - result: [{COL3: 1, S: 1}, {COL3: 2, S: 2}, {COL3: 100, S: 1}, {COL3: 200, S: 2}] + - maxRows: 1 + - result: [{COL3: 1, S: 1}] + - result: [{COL3: 2, S: 2}] + - result: [{COL3: 100, S: 1}] + - result: [{COL3: 200, S: 2}] + - result: [] - - query: select col3, sum(col2) as s from t2 use index (mv9) where col1 = 1 group by col1, col3 order by col3 desc; - supported_version: 4.1.9.0 @@ -269,8 +273,12 @@ test_block: # Cannot use FORCE_CONTINUATIONS with older versions due to: https://github.com/FoundationDB/fdb-record-layer/issues/3096 # (Extra values being produced after exhausting source of an aggregate cursor) # Can remove once we do not care about backwards compatibility before 4.1.9.0 - - maxRows: 0 - - result: [{COL3: 200, S: 2}, {COL3: 100, S: 1}, {COL3: 2, S: 2}, {COL3: 1, S: 1}] + - maxRows: 1 + - result: [{COL3: 200, S: 2}] + - result: [{COL3: 100, S: 1}] + - result: [{COL3: 2, S: 2}] + - result: [{COL3: 1, S: 1}] + - result: [] # - # # grouping by constant is not yet supported. # - query: select sum(col2) from t1 group by 3,2,1; From 78bea95e63652ed3f35f89bd25a424f154552382 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 29 May 2025 14:33:38 -0700 Subject: [PATCH 04/15] save --- .../foundationdb/KeyValueCursorTest.java | 2 +- .../query/FDBStreamAggregationTest.java | 19 ++++++++ .../recordlayer/query/GroupByQueryTests.java | 47 +++++++++++++++++++ 3 files changed, 67 insertions(+), 1 deletion(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java index d426804c61..1fb78f169e 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java @@ -189,7 +189,7 @@ public void lastRow() { .setRange(TupleRange.ALL) .setContinuation(cursor.getNext().getContinuation().toBytes()) .setScanProperties(ScanProperties.FORWARD_SCAN) - .setSerializationMode(KeyValueCursorBase.SerializationMode.TO_NEW) + .setSerializationMode(KeyValueCursorBase.SerializationMode.TO_OLD) .build(); assertEquals(0, (int)cursor.getCount().join()); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java index 323693b340..350dd91cf1 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java @@ -49,6 +49,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; @@ -97,6 +98,24 @@ public void setup() throws Exception { populateDB(5); } + @Test + void test() { + try (final var context = openContext()) { + openSimpleRecordStore(context, NO_HOOK); + + // select col3, sum(col2) as s from t2 use index (mv9) where col1 = 1 group by col1, col3 order by col3 desc; + final var plan = + new AggregationPlanBuilder(recordStore.getRecordMetaData(), "MySimpleRecord") + .withAggregateValue("num_value_2", value -> new NumericAggregationValue.Sum(NumericAggregationValue.PhysicalOperator.SUM_I, value)) + .withGroupCriterion("num_value_3_indexed") + .withGroupCriterion("str_value_indexed") + .build(false); + + final var result = executePlanWithRowLimit(plan, 1); + // assertResults(this::assertResultFlattened, result, resultOf(0, 1), resultOf(1, 5), resultOf(2, 9)); + } + } + @ParameterizedTest(name = "[{displayName}-{index}] {0}") @MethodSource("provideArguments") void noAggregateGroupByNone(final boolean useNestedResult, final int rowLimit) { diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java index e10e846eff..baa0f31e0e 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.relational.recordlayer.query; +import com.apple.foundationdb.relational.api.Continuation; import com.apple.foundationdb.relational.api.RelationalResultSet; import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalExtension; import com.apple.foundationdb.relational.recordlayer.Utils; @@ -33,6 +34,7 @@ import org.junit.jupiter.api.extension.RegisterExtension; import java.net.URI; +import java.util.Base64; import static com.apple.foundationdb.relational.recordlayer.query.QueryTestUtils.insertT1Record; import static com.apple.foundationdb.relational.recordlayer.query.QueryTestUtils.insertT1RecordColAIsNull; @@ -79,6 +81,51 @@ void isNullPredicateUsesGroupIndex() throws Exception { } } + @Test + void test() throws Exception { + final String schemaTemplate = + "CREATE TABLE T1(pk bigint, a bigint, b bigint, c bigint, PRIMARY KEY(pk))\n" + + "create index mv9 as select a, max(b), c from T1 group by a, c order by a, max(b), c"; + try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { + try (var statement = ddl.setSchemaAndGetConnection().createStatement()) { + insertT1Record(statement, 1, 1, 1, 100); + insertT1Record(statement, 2, 1, 1, 1); + insertT1Record(statement, 3, 1, 2, 2); + insertT1Record(statement, 4, 1, 2, 200); + insertT1Record(statement, 5, 2, 1, 200); + insertT1Record(statement, 6, 2, 1, 3); + insertT1Record(statement, 7, 2, 1, 400); + insertT1Record(statement, 8, 2, 1, 400); + insertT1Record(statement, 9, 2, 1, 400); + //statement.setMaxRows(1); + Continuation continuation; + try (final RelationalResultSet resultSet = statement.executeQuery("select c, sum(b) as s from t1 use index (mv9) where a = 1 group by a, c order by c desc")) { + //Assertions.assertEquals(200L, resultSet.getLong(1)); + //continuation = resultSet.getContinuation(); + + ResultSetAssert.assertThat(resultSet).hasNextRow() + .isRowExactly(200L, 2L) + .hasNextRow() + .isRowExactly(100L, 1L) + .hasNextRow() + .isRowExactly(2L, 2L) + .hasNextRow() + .isRowExactly(1L, 1L) + .hasNoNextRow(); + } + /* + Assertions.assertTrue(statement.execute("EXECUTE CONTINUATION " + Base64.getEncoder().encodeToString(continuation.serialize())), "Did not return a result set from a select statement!"); + try (final RelationalResultSet resultSet = statement.getResultSet()) { + Assertions.assertEquals(100L, resultSet.getLong(1)); + continuation = resultSet.getContinuation(); + } + + */ + } + + } + } + @Test void groupByClauseWithPredicateWorks() throws Exception { final String schemaTemplate = From aefa82cd48d978b69c9aad7728ebbb5da6d04651 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Fri, 30 May 2025 22:39:10 -0700 Subject: [PATCH 05/15] save --- .../foundationdb/KeyValueCursorBase.java | 2 +- .../plan/RecordQueryPlannerConfiguration.java | 11 ++++++ .../main/proto/record_planner_config.proto | 6 +++ .../RecordQueryIndexPlanWithOverScanTest.java | 2 + .../recordlayer/query/GroupByQueryTests.java | 37 ++++++++----------- 5 files changed, 36 insertions(+), 22 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java index 429703b365..bfbf33f413 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java @@ -244,7 +244,7 @@ public abstract static class Builder> { private KeySelector begin; private KeySelector end; // default to be old now - protected SerializationMode serializationMode = SerializationMode.TO_OLD; + protected SerializationMode serializationMode = SerializationMode.TO_NEW; protected Builder(@Nonnull Subspace subspace) { this.subspace = subspace; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java index 0a64b89981..87b003a480 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.IndexFetchMethod; import com.apple.foundationdb.record.RecordPlannerConfigurationProto; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.plan.cascades.CascadesRule; import com.apple.foundationdb.record.query.plan.cascades.PlannerRule; @@ -434,6 +435,16 @@ public Builder() { this.protoBuilder = RecordPlannerConfigurationProto.PlannerConfiguration.newBuilder(); } + @Nonnull + public Builder setKeyValueContinuationSerializationMode(@Nonnull KeyValueCursorBase.SerializationMode serializationMode) { + if (serializationMode == KeyValueCursorBase.SerializationMode.TO_OLD) { + protoBuilder.setKeyValueCursorContinuationSerializationMode(RecordPlannerConfigurationProto.PlannerConfiguration.KeyValueCursorContinuationSerializationMode.TO_OLD); + } else { + protoBuilder.setKeyValueCursorContinuationSerializationMode(RecordPlannerConfigurationProto.PlannerConfiguration.KeyValueCursorContinuationSerializationMode.TO_NEW); + } + return this; + } + @Nonnull public Builder setIndexScanPreference(@Nonnull QueryPlanner.IndexScanPreference indexScanPreference) { protoBuilder.setIndexScanPreference(SCAN_PREFERENCE_BI_MAP.get(indexScanPreference)); diff --git a/fdb-record-layer-core/src/main/proto/record_planner_config.proto b/fdb-record-layer-core/src/main/proto/record_planner_config.proto index 23b1a4f029..7156d9ff10 100644 --- a/fdb-record-layer-core/src/main/proto/record_planner_config.proto +++ b/fdb-record-layer-core/src/main/proto/record_planner_config.proto @@ -36,6 +36,11 @@ message PlannerConfiguration { USE_REMOTE_FETCH_WITH_FALLBACK = 2; } + enum KeyValueCursorContinuationSerializationMode { + TO_OLD = 0; + TO_NEW = 1; + } + message SortConfiguration { optional bool shouldAllowNonIndexSort = 1; } @@ -54,4 +59,5 @@ message PlannerConfiguration { optional int32 maxNumReplansForInToJoin = 12; optional int32 orToUnionMaxNumConjuncts = 13; optional int32 maxNumReplansForInUnion = 14; + optional KeyValueCursorContinuationSerializationMode keyValueCursorContinuationSerializationMode = 15; } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java index 98763c8670..ff2aecf2a4 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java @@ -34,6 +34,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStore; import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.provider.foundationdb.query.FDBRecordStoreQueryTestBase; import com.apple.foundationdb.record.query.ParameterRelationshipGraph; import com.apple.foundationdb.record.query.RecordQuery; @@ -78,6 +79,7 @@ class RecordQueryIndexPlanWithOverScanTest extends FDBRecordStoreQueryTestBase { private static final RecordQueryPlannerConfiguration plannerConfiguration = RecordQueryPlannerConfiguration.builder() .setIndexScanPreference(QueryPlanner.IndexScanPreference.PREFER_INDEX) .setAttemptFailedInJoinAsOr(true) + .setKeyValueContinuationSerializationMode(KeyValueCursorBase.SerializationMode.TO_NEW) .setComplexityThreshold(RecordQueryPlanner.DEFAULT_COMPLEXITY_THRESHOLD) .addValueIndexOverScanNeeded("MySimpleRecord$str_value_indexed") .addValueIndexOverScanNeeded("compoundIndex") diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java index baa0f31e0e..63507c1789 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java @@ -34,6 +34,7 @@ import org.junit.jupiter.api.extension.RegisterExtension; import java.net.URI; +import java.util.Arrays; import java.util.Base64; import static com.apple.foundationdb.relational.recordlayer.query.QueryTestUtils.insertT1Record; @@ -85,7 +86,7 @@ void isNullPredicateUsesGroupIndex() throws Exception { void test() throws Exception { final String schemaTemplate = "CREATE TABLE T1(pk bigint, a bigint, b bigint, c bigint, PRIMARY KEY(pk))\n" + - "create index mv9 as select a, max(b), c from T1 group by a, c order by a, max(b), c"; + "create index mv1 as select count(*) from t1\n"; try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { try (var statement = ddl.setSchemaAndGetConnection().createStatement()) { insertT1Record(statement, 1, 1, 1, 100); @@ -97,32 +98,26 @@ void test() throws Exception { insertT1Record(statement, 7, 2, 1, 400); insertT1Record(statement, 8, 2, 1, 400); insertT1Record(statement, 9, 2, 1, 400); - //statement.setMaxRows(1); + statement.setMaxRows(1); Continuation continuation; - try (final RelationalResultSet resultSet = statement.executeQuery("select c, sum(b) as s from t1 use index (mv9) where a = 1 group by a, c order by c desc")) { - //Assertions.assertEquals(200L, resultSet.getLong(1)); - //continuation = resultSet.getContinuation(); - + try (final RelationalResultSet resultSet = statement.executeQuery("select count(*) from t1")) { ResultSetAssert.assertThat(resultSet).hasNextRow() - .isRowExactly(200L, 2L) - .hasNextRow() - .isRowExactly(100L, 1L) - .hasNextRow() - .isRowExactly(2L, 2L) - .hasNextRow() - .isRowExactly(1L, 1L) + .isRowExactly(9L) .hasNoNextRow(); - } - /* - Assertions.assertTrue(statement.execute("EXECUTE CONTINUATION " + Base64.getEncoder().encodeToString(continuation.serialize())), "Did not return a result set from a select statement!"); - try (final RelationalResultSet resultSet = statement.getResultSet()) { - Assertions.assertEquals(100L, resultSet.getLong(1)); continuation = resultSet.getContinuation(); } - - */ + System.out.println("continuation at beginning:" + continuation.atBeginning()); + try (final var preparedStatement = ddl.setSchemaAndGetConnection().prepareStatement("EXECUTE CONTINUATION ?param")) { + preparedStatement.setMaxRows(1); + preparedStatement.setBytes("param", continuation.serialize()); + try (final var resultSet = preparedStatement.executeQuery()) { + ResultSetAssert.assertThat(resultSet) + .hasNoNextRow(); + continuation = resultSet.getContinuation(); + } + } + Assertions.assertTrue(continuation.atEnd()); } - } } From 3fe7ba0b47192414c70e957ac056b443b51d7366 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 4 Jun 2025 16:55:53 -0700 Subject: [PATCH 06/15] save --- .../foundationdb/KeyValueCursorBase.java | 23 +++++++++++++++++-- .../plan/plans/RecordQueryIndexPlan.java | 13 ++++++++--- .../RecordQueryIndexPlanWithOverScanTest.java | 12 +++++++--- 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java index bfbf33f413..8545678522 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java @@ -142,7 +142,7 @@ private RecordCursorContinuation continuationHelper() { return new Continuation(lastKey, prefixLength, serializationMode); } - private static class Continuation implements RecordCursorContinuation { + public static class Continuation implements RecordCursorContinuation { @Nullable private final byte[] lastKey; private final int prefixLength; @@ -190,6 +190,23 @@ public byte[] toBytes() { } } + @Nullable + public byte[] getInnerContinuationInBytes() { + if (lastKey == null) { + return null; + } + return Arrays.copyOfRange(lastKey, prefixLength, lastKey.length); + } + + @Nonnull + public ByteString getInnerContinuationInByteString() { + if (lastKey == null) { + return ByteString.EMPTY; + } + ByteString base = ZeroCopyByteString.wrap(lastKey); + return base.substring(prefixLength, lastKey.length); + } + @Nonnull private RecordCursorProto.KeyValueCursorContinuation toProto() { RecordCursorProto.KeyValueCursorContinuation.Builder builder = RecordCursorProto.KeyValueCursorContinuation.newBuilder(); @@ -197,7 +214,8 @@ private RecordCursorProto.KeyValueCursorContinuation toProto() { builder.setIsEnd(true); } else { ByteString base = ZeroCopyByteString.wrap(lastKey); - builder.setContinuation(base.substring(prefixLength, lastKey.length)).setIsEnd(false); + builder.setContinuation(base.substring(prefixLength, lastKey.length)) + .setIsEnd(false); } return builder.build(); } @@ -293,6 +311,7 @@ protected void prepare() { RecordCursorProto.KeyValueCursorContinuation keyValueCursorContinuation = RecordCursorProto.KeyValueCursorContinuation.parseFrom(continuation); realContinuation = keyValueCursorContinuation.getContinuation().toByteArray(); } catch (InvalidProtocolBufferException ex) { + System.out.println("InvalidProtocolBufferException hit"); realContinuation = continuation; } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java index a3dd4539ad..b4882eb504 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java @@ -37,6 +37,7 @@ import com.apple.foundationdb.record.RecordCursor; import com.apple.foundationdb.record.RecordCursorContinuation; import com.apple.foundationdb.record.RecordCursorEndContinuation; +import com.apple.foundationdb.record.RecordCursorProto; import com.apple.foundationdb.record.RecordCursorResult; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.ScanProperties; @@ -57,6 +58,7 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; import com.apple.foundationdb.record.provider.foundationdb.IndexScanRange; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.provider.foundationdb.MultidimensionalIndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.UnsupportedRemoteFetchIndexException; import com.apple.foundationdb.record.query.plan.AvailableFields; @@ -759,7 +761,12 @@ public RecordCursorContinuation wrapContinuation(@Nonnull final RecordCursorCont if (continuation.isEnd()) { return continuation; } - final byte[] continuationBytes = continuation.toBytes(); + byte[] continuationBytes; + if (continuation instanceof KeyValueCursorBase.Continuation) { + continuationBytes = ((KeyValueCursorBase.Continuation) continuation).getInnerContinuationInBytes(); + } else { + continuationBytes = continuation.toBytes(); + } if (continuationBytes != null && ByteArrayUtil.startsWith(continuationBytes, prefixBytes)) { // Strip away the prefix. Note that ByteStrings re-use the underlying ByteArray, so this can // save a copy. @@ -790,7 +797,7 @@ public byte[] toBytes() { if (bytes == null) { synchronized (this) { if (bytes == null) { - byte[] baseContinuationBytes = baseContinuation.toBytes(); + byte[] baseContinuationBytes = baseContinuation instanceof KeyValueCursorBase.Continuation ? ((KeyValueCursorBase.Continuation) baseContinuation).getInnerContinuationInBytes() : baseContinuation.toBytes(); if (baseContinuationBytes == null) { return null; } @@ -804,7 +811,7 @@ public byte[] toBytes() { @Nonnull @Override public ByteString toByteString() { - return baseContinuation.toByteString().substring(prefixLength); + return (baseContinuation instanceof KeyValueCursorBase.Continuation) ? ((KeyValueCursorBase.Continuation) baseContinuation).getInnerContinuationInByteString().substring(prefixLength) : baseContinuation.toByteString().substring(prefixLength); } @Override diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java index ff2aecf2a4..fc32f94233 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java @@ -44,6 +44,7 @@ import com.apple.foundationdb.record.query.plan.RecordQueryPlannerConfiguration; import com.apple.test.BooleanSource; import com.apple.test.Tags; +import com.google.protobuf.ByteString; import com.google.protobuf.Message; import org.hamcrest.Matcher; import org.junit.jupiter.api.Tag; @@ -66,6 +67,7 @@ import static org.hamcrest.Matchers.allOf; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -218,7 +220,7 @@ void withLimits(boolean reverse, int limit) throws Exception { ExecuteProperties executeProperties = ExecuteProperties.newBuilder() .setDefaultCursorStreamingMode(CursorStreamingMode.WANT_ALL) - .setReturnedRowLimit(limit) + .setReturnedRowLimit(1) .build(); EvaluationContext evaluationContext = EvaluationContext.EMPTY .withBinding("str_val", "threven") @@ -250,8 +252,12 @@ private static RecordCursorContinuation assertSameResults(@Nonnull FDBRecordStor do { indexResult = indexCursor.getNext(); overscanResult = overscanCursor.getNext(); - assertEquals(indexResult.getContinuation().toByteString(), overscanResult.getContinuation().toByteString(), "Continuation byte strings should match"); - assertArrayEquals(indexResult.getContinuation().toBytes(), overscanResult.getContinuation().toBytes(), "Continuation byte arrays should match"); + + ByteString indexResultContinuationInByteString = indexResult.getContinuation() instanceof KeyValueCursorBase.Continuation ? ((KeyValueCursorBase.Continuation) indexResult.getContinuation()).getInnerContinuationInByteString() : indexResult.getContinuation().toByteString(); + byte[] indexResultContinuationInBytes = indexResult.getContinuation() instanceof KeyValueCursorBase.Continuation ? ((KeyValueCursorBase.Continuation) indexResult.getContinuation()).getInnerContinuationInBytes() : indexResult.getContinuation().toBytes(); + + assertEquals(indexResultContinuationInByteString, overscanResult.getContinuation().toByteString(), "Continuation byte strings should match"); + assertArrayEquals(indexResultContinuationInBytes, overscanResult.getContinuation().toBytes(), "Continuation byte arrays should match"); assertEquals(indexResult.hasNext(), overscanResult.hasNext(), "Overscan cursor should have next if index result has next"); if (indexResult.hasNext()) { From 44a5cb906169a645e17975b13ea6d3aa5b85db6d Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 11 Jun 2025 22:20:52 -0700 Subject: [PATCH 07/15] clean --- .../foundationdb/KeyValueCursorBase.java | 15 +++++++++++++ .../plan/plans/RecordQueryIndexPlan.java | 8 ++----- .../query/FDBStreamAggregationTest.java | 21 ------------------- .../RecordQueryIndexPlanWithOverScanTest.java | 6 ++---- .../recordlayer/query/GroupByQueryTests.java | 2 -- 5 files changed, 19 insertions(+), 33 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java index 61b8c3b536..31aae73759 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java @@ -28,6 +28,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; import com.apple.foundationdb.async.AsyncIterator; +import com.apple.foundationdb.record.ByteArrayContinuation; import com.apple.foundationdb.record.CursorStreamingMode; import com.apple.foundationdb.record.EndpointType; import com.apple.foundationdb.record.KeyRange; @@ -40,6 +41,8 @@ import com.apple.foundationdb.record.cursors.AsyncIteratorCursor; import com.apple.foundationdb.record.cursors.BaseCursor; import com.apple.foundationdb.record.cursors.CursorLimitManager; +import com.apple.foundationdb.record.cursors.aggregate.AggregateCursor; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; import com.apple.foundationdb.subspace.Subspace; import com.apple.foundationdb.tuple.ByteArrayUtil2; import com.apple.foundationdb.tuple.Tuple; @@ -209,6 +212,18 @@ public ByteString getInnerContinuationInByteString() { return base.substring(prefixLength, lastKey.length); } + public static byte[] fromRawBytes(@Nonnull byte[] rawBytes, SerializationMode serializationMode) { + if (serializationMode == SerializationMode.TO_OLD) { + return rawBytes; + } + try { + RecordCursorProto.KeyValueCursorContinuation continuationProto = RecordCursorProto.KeyValueCursorContinuation.parseFrom(rawBytes); + return continuationProto.getContinuation().toByteArray(); + } catch (InvalidProtocolBufferException ipbe) { + return rawBytes; + } + } + @Nonnull private RecordCursorProto.KeyValueCursorContinuation toProto() { RecordCursorProto.KeyValueCursorContinuation.Builder builder = RecordCursorProto.KeyValueCursorContinuation.newBuilder(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java index 9884fd03ce..252bf0c439 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java @@ -760,12 +760,8 @@ public byte[] unwrapContinuation(@Nullable final byte[] continuation) { return null; } // Add the prefix back to the inner continuation - try { - RecordCursorProto.KeyValueCursorContinuation keyValueCursorContinuation = RecordCursorProto.KeyValueCursorContinuation.parseFrom(continuation); - return ByteArrayUtil.join(prefixBytes, keyValueCursorContinuation.getContinuation().toByteArray()); - } catch (InvalidProtocolBufferException ex) { - return ByteArrayUtil.join(prefixBytes, continuation); - } + byte[] innerContinuation = KeyValueCursorBase.Continuation.fromRawBytes(continuation, KeyValueCursorBase.SerializationMode.TO_NEW); + return ByteArrayUtil.join(prefixBytes, innerContinuation); } @Override diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java index 5c0f7ad6b2..d42052619a 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java @@ -111,27 +111,6 @@ public void setup() throws Exception { populateDB(5); } - /* - @Test - void test() { - try (final var context = openContext()) { - openSimpleRecordStore(context, NO_HOOK); - - // select col3, sum(col2) as s from t2 use index (mv9) where col1 = 1 group by col1, col3 order by col3 desc; - final var plan = - new AggregationPlanBuilder(recordStore.getRecordMetaData(), "MySimpleRecord") - .withAggregateValue("num_value_2", value -> new NumericAggregationValue.Sum(NumericAggregationValue.PhysicalOperator.SUM_I, value)) - .withGroupCriterion("num_value_3_indexed") - .withGroupCriterion("str_value_indexed") - .build(false); - - final var result = executePlanWithRowLimit(plan, 1); - // assertResults(this::assertResultFlattened, result, resultOf(0, 1), resultOf(1, 5), resultOf(2, 9)); - } - } - - */ - @ParameterizedTest(name = "[{displayName}-{index}] {0}") @MethodSource("provideArguments") void noAggregateGroupByNone(final boolean useNestedResult, final RecordQueryStreamingAggregationPlan.SerializationMode serializationMode, final int rowLimit) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java index b3c790e763..e505f40753 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java @@ -253,16 +253,14 @@ private static RecordCursorContinuation assertSameResults(@Nonnull FDBRecordStor indexResult = indexCursor.getNext(); overscanResult = overscanCursor.getNext(); - System.out.println("indexResult continuation class:" + indexResult.getContinuation().getClass()); ByteString indexResultContinuationInByteString = indexResult.getContinuation() instanceof KeyValueCursorBase.Continuation ? ((KeyValueCursorBase.Continuation) indexResult.getContinuation()).getInnerContinuationInByteString() : indexResult.getContinuation().toByteString(); byte[] indexResultContinuationInBytes = indexResult.getContinuation() instanceof KeyValueCursorBase.Continuation ? ((KeyValueCursorBase.Continuation) indexResult.getContinuation()).getInnerContinuationInBytes() : indexResult.getContinuation().toBytes(); - // assertEquals(indexResultContinuationInByteString, overscanResult.getContinuation().toByteString(), "Continuation byte strings should match"); - //assertArrayEquals(indexResultContinuationInBytes, overscanResult.getContinuation().toBytes(), "Continuation byte arrays should match"); + assertEquals(indexResultContinuationInByteString, overscanResult.getContinuation().toByteString(), "Continuation byte strings should match"); + assertArrayEquals(indexResultContinuationInBytes, overscanResult.getContinuation().toBytes(), "Continuation byte arrays should match"); assertEquals(indexResult.hasNext(), overscanResult.hasNext(), "Overscan cursor should have next if index result has next"); if (indexResult.hasNext()) { - System.out.println("index result:" + indexResult.get().getRecord() + " overscan result:" + overscanResult.get().getRecord()); assertEquals(indexResult.get().getRecord(), overscanResult.get().getRecord(), "Result returned via overscan cursor should match regular cursor"); } else { assertEquals(indexResult.getNoNextReason(), overscanResult.getNoNextReason(), "Overscan cursor should have same no next reason as index result"); diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java index 13cbd98ace..245b61a423 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java @@ -20,8 +20,6 @@ package com.apple.foundationdb.relational.recordlayer.query; -import com.apple.foundationdb.record.RecordCursorEndContinuation; -import com.apple.foundationdb.record.RecordCursorStartContinuation; import com.apple.foundationdb.relational.api.Continuation; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.RelationalResultSet; From cb40b881f858a624a97c0d4c955777da193ff236 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 12 Jun 2025 12:07:33 -0700 Subject: [PATCH 08/15] save --- .../foundationdb/KeyValueCursorTest.java | 64 ------------------- .../recordlayer/query/GroupByQueryTests.java | 39 ----------- 2 files changed, 103 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java index 1fb78f169e..1854d84083 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorTest.java @@ -133,70 +133,6 @@ public void all(KeyValueCursorBase.SerializationMode serializationMode) { }); } - @Test - public void allOldThanNew() { - fdb.run(context -> { - KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) - .setContext(context) - .setRange(TupleRange.ALL) - .setContinuation(null) - .setScanProperties(ScanProperties.FORWARD_SCAN) - .setSerializationMode(KeyValueCursorBase.SerializationMode.TO_OLD) - .build(); - for (int i = 0; i < 5; i++) { - for (int j = 0; j < 5; j++) { - KeyValue kv = cursor.getNext().get(); - assertArrayEquals(subspace.pack(Tuple.from(i, j)), kv.getKey()); - assertArrayEquals(Tuple.from(i, j).pack(), kv.getValue()); - } - } - assertThat(cursor.getNext().hasNext(), is(false)); - - cursor = KeyValueCursor.Builder.withSubspace(subspace) - .setContext(context) - .setRange(TupleRange.ALL) - .setContinuation(null) - .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(10).build())) - .setSerializationMode(KeyValueCursorBase.SerializationMode.TO_OLD) - .build(); - assertEquals(10, (int)cursor.getCount().join()); - cursor = KeyValueCursor.Builder.withSubspace(subspace) - .setContext(context) - .setRange(TupleRange.ALL) - .setContinuation(cursor.getNext().getContinuation().toBytes()) - .setScanProperties(ScanProperties.FORWARD_SCAN) - .setSerializationMode(KeyValueCursorBase.SerializationMode.TO_NEW) - .build(); - assertEquals(15, (int)cursor.getCount().join()); - - return null; - }); - } - - @Test - public void lastRow() { - fdb.run(context -> { - KeyValueCursor cursor = KeyValueCursor.Builder.withSubspace(subspace) - .setContext(context) - .setRange(TupleRange.ALL) - .setContinuation(null) - .setScanProperties(new ScanProperties(ExecuteProperties.newBuilder().setReturnedRowLimit(25).build())) - .setSerializationMode(KeyValueCursorBase.SerializationMode.TO_OLD) - .build(); - assertEquals(25, (int)cursor.getCount().join()); - cursor = KeyValueCursor.Builder.withSubspace(subspace) - .setContext(context) - .setRange(TupleRange.ALL) - .setContinuation(cursor.getNext().getContinuation().toBytes()) - .setScanProperties(ScanProperties.FORWARD_SCAN) - .setSerializationMode(KeyValueCursorBase.SerializationMode.TO_OLD) - .build(); - assertEquals(0, (int)cursor.getCount().join()); - - return null; - }); - } - @ParameterizedTest @EnumSource(KeyValueCursorBase.SerializationMode.class) public void beginsWith(KeyValueCursorBase.SerializationMode serializationMode) { diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java index 245b61a423..e3e3cb85ff 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java @@ -245,45 +245,6 @@ void isNullPredicateUsesGroupIndex() throws Exception { } } - @Test - void test() throws Exception { - final String schemaTemplate = - "CREATE TABLE T1(pk bigint, a bigint, b bigint, c bigint, PRIMARY KEY(pk))\n" + - "create index mv1 as select count(*) from t1\n"; - try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { - try (var statement = ddl.setSchemaAndGetConnection().createStatement()) { - insertT1Record(statement, 1, 1, 1, 100); - insertT1Record(statement, 2, 1, 1, 1); - insertT1Record(statement, 3, 1, 2, 2); - insertT1Record(statement, 4, 1, 2, 200); - insertT1Record(statement, 5, 2, 1, 200); - insertT1Record(statement, 6, 2, 1, 3); - insertT1Record(statement, 7, 2, 1, 400); - insertT1Record(statement, 8, 2, 1, 400); - insertT1Record(statement, 9, 2, 1, 400); - statement.setMaxRows(1); - Continuation continuation; - try (final RelationalResultSet resultSet = statement.executeQuery("select count(*) from t1")) { - ResultSetAssert.assertThat(resultSet).hasNextRow() - .isRowExactly(9L) - .hasNoNextRow(); - continuation = resultSet.getContinuation(); - } - System.out.println("continuation at beginning:" + continuation.atBeginning()); - try (final var preparedStatement = ddl.setSchemaAndGetConnection().prepareStatement("EXECUTE CONTINUATION ?param")) { - preparedStatement.setMaxRows(1); - preparedStatement.setBytes("param", continuation.serialize()); - try (final var resultSet = preparedStatement.executeQuery()) { - ResultSetAssert.assertThat(resultSet) - .hasNoNextRow(); - continuation = resultSet.getContinuation(); - } - } - Assertions.assertTrue(continuation.atEnd()); - } - } - } - @Test void groupByClauseWithPredicateWorks() throws Exception { final String schemaTemplate = From 8d523c133a0109bf01f53c40e016cb8e3ab1a9e0 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 12 Jun 2025 20:20:44 -0700 Subject: [PATCH 09/15] add planner configuration --- .../record/query/plan/RecordQueryPlanner.java | 4 +- .../plan/RecordQueryPlannerConfiguration.java | 9 ++++- .../AggregateIndexMatchCandidate.java | 3 +- .../ValueIndexScanMatchCandidate.java | 6 ++- .../WindowedIndexScanMatchCandidate.java | 6 ++- .../plan/plans/RecordQueryIndexPlan.java | 34 ++++++++++++---- .../foundationdb/QueryPlanCursorTest.java | 8 ++-- .../foundationdb/RecordTypeKeyTest.java | 2 +- .../MultidimensionalIndexTestBase.java | 40 +++++++++++++------ .../SlowMultidimensionalIndexTest.java | 4 +- .../limits/FDBRecordStoreLimitTestBase.java | 7 ++-- .../limits/FDBRecordStoreScanLimitTest.java | 3 +- .../query/FDBAndQueryToIntersectionTest.java | 9 +++-- .../query/FDBOrQueryToUnionTest.java | 9 +++-- ...ueryPlanStructuralInstrumentationTest.java | 5 ++- .../matchers/ExpressionMatcherTest.java | 14 ++++--- .../plan/explain/ExplainPlanVisitorTest.java | 3 +- .../planning/RecordQueryPlanEqualityTest.java | 3 +- .../RecordQueryIndexPlanWithOverScanTest.java | 2 +- .../serialization/PlanSerializationTest.java | 7 +++- 20 files changed, 119 insertions(+), 59 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlanner.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlanner.java index f1bcd08a7f..83112e6fd6 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlanner.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlanner.java @@ -1765,7 +1765,7 @@ private RecordQueryPlan planScan(@Nonnull CandidateScan candidateScan, fetchIndexRecords == FetchIndexRecords.PRIMARY_KEY ? getConfiguration().getIndexFetchMethod() : IndexFetchMethod.SCAN_AND_FETCH; - plan = new RecordQueryIndexPlan(candidateScan.index.getName(), candidateScan.planContext.commonPrimaryKey, indexScanParameters, indexFetchMethod, fetchIndexRecords, candidateScan.reverse, strictlySorted); + plan = new RecordQueryIndexPlan(candidateScan.index.getName(), candidateScan.planContext.commonPrimaryKey, indexScanParameters, indexFetchMethod, fetchIndexRecords, candidateScan.reverse, strictlySorted, configuration.getKeyValueCursorSerializationMode()); possibleTypes = getPossibleTypes(candidateScan.index); } // Add a type filter if the query plan might return records of more types than the query specified @@ -2158,7 +2158,7 @@ public RecordQueryCoveringIndexPlan planCoveringAggregateIndex(@Nonnull RecordQu RecordQueryIndexPlan plan = (RecordQueryIndexPlan)scoredPlan.getPlan(); IndexScanParameters scanParameters = new IndexScanComparisons(IndexScanType.BY_GROUP, plan.getScanComparisons()); - plan = new RecordQueryIndexPlan(plan.getIndexName(), scanParameters, plan.isReverse()); + plan = new RecordQueryIndexPlan(plan.getIndexName(), scanParameters, plan.isReverse(), configuration.getKeyValueCursorSerializationMode()); return new RecordQueryCoveringIndexPlan(plan, recordType.getName(), AvailableFields.NO_FIELDS, builder.build()); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java index 87b003a480..fbb1439496 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java @@ -66,6 +66,7 @@ public class RecordQueryPlannerConfiguration { private static final long PLAN_OTHER_ATTEMPT_FULL_FILTER_MASK = 1L << 9; private static final long NORMALIZE_NESTED_FIELDS_MASK = 1L << 10; private static final long OMIT_PRIMARY_KEY_IN_ORDERING_KEY_FOR_IN_UNION_MASK = 1L << 11; + private static final long KEYVALUE_CURSOR_CONTINUATION_SERIALIZATION_MODE = 1L << 12; @Nonnull private final RecordPlannerConfigurationProto.PlannerConfiguration proto; @@ -229,6 +230,10 @@ public boolean shouldOptimizeForRequiredResults() { return flagSet(OPTIMIZE_FOR_REQUIRED_RESULTS_MASK); } + public KeyValueCursorBase.SerializationMode getKeyValueCursorSerializationMode() { + return flagSet(KEYVALUE_CURSOR_CONTINUATION_SERIALIZATION_MODE) ? KeyValueCursorBase.SerializationMode.TO_NEW : KeyValueCursorBase.SerializationMode.TO_OLD; + } + /** * Return the size limit of the cascades planner task queue. * @return the maximum size of the queue. 0 means "unbound" (the default). Trying to add a task beyond the maximum size will fail the planning. @@ -436,11 +441,13 @@ public Builder() { } @Nonnull - public Builder setKeyValueContinuationSerializationMode(@Nonnull KeyValueCursorBase.SerializationMode serializationMode) { + public Builder setKeyValueCursorContinuationSerializationMode(@Nonnull KeyValueCursorBase.SerializationMode serializationMode) { if (serializationMode == KeyValueCursorBase.SerializationMode.TO_OLD) { protoBuilder.setKeyValueCursorContinuationSerializationMode(RecordPlannerConfigurationProto.PlannerConfiguration.KeyValueCursorContinuationSerializationMode.TO_OLD); + updateFlags(false, KEYVALUE_CURSOR_CONTINUATION_SERIALIZATION_MODE); } else { protoBuilder.setKeyValueCursorContinuationSerializationMode(RecordPlannerConfigurationProto.PlannerConfiguration.KeyValueCursorContinuationSerializationMode.TO_NEW); + updateFlags(true, KEYVALUE_CURSOR_CONTINUATION_SERIALIZATION_MODE); } return this; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexMatchCandidate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexMatchCandidate.java index 11b5dcc4e6..ac26ab3efb 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexMatchCandidate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexMatchCandidate.java @@ -383,7 +383,8 @@ public RecordQueryPlan toEquivalentPlan(@Nonnull final PartialMatch partialMatch false, partialMatch.getMatchCandidate(), baseRecordType, - QueryPlanConstraint.tautology()); + QueryPlanConstraint.tautology(), + planContext.getPlannerConfiguration().getKeyValueCursorSerializationMode()); return new RecordQueryAggregateIndexPlan(aggregateIndexScan, recordTypes.get(0).getName(), diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java index d861839d57..6d6da35a1c 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java @@ -245,7 +245,8 @@ public RecordQueryPlan toEquivalentPlan(@Nonnull final PartialMatch partialMatch false, partialMatch.getMatchCandidate(), baseRecordType, - matchInfo.getConstraint())); + matchInfo.getConstraint(), + planContext.getPlannerConfiguration().getKeyValueCursorSerializationMode())); } @Nonnull @@ -272,7 +273,8 @@ private Optional tryFetchCoveringIndexScan(@Nonnull final Parti false, partialMatch.getMatchCandidate(), baseRecordType, - partialMatch.getRegularMatchInfo().getConstraint()); + partialMatch.getRegularMatchInfo().getConstraint(), + planContext.getPlannerConfiguration().getKeyValueCursorSerializationMode()); final var coveringIndexPlan = new RecordQueryCoveringIndexPlan(indexPlan, indexEntryToLogicalRecord.getQueriedRecordType().getName(), diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java index abab9990bd..9d23b6e611 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java @@ -403,7 +403,8 @@ public RecordQueryPlan toEquivalentPlan(@Nonnull final PartialMatch partialMatch false, partialMatch.getMatchCandidate(), baseRecordType, - QueryPlanConstraint.tautology())); + QueryPlanConstraint.tautology(), + planContext.getPlannerConfiguration().getKeyValueCursorSerializationMode())); } @Nonnull @@ -429,7 +430,8 @@ private Optional tryFetchCoveringIndexScan(@Nonnull final Parti false, partialMatch.getMatchCandidate(), baseRecordType, - QueryPlanConstraint.tautology()); + QueryPlanConstraint.tautology(), + planContext.getPlannerConfiguration().getKeyValueCursorSerializationMode()); final var coveringIndexPlan = new RecordQueryCoveringIndexPlan(indexPlan, indexEntryToLogicalRecord.getQueriedRecordType().getName(), diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java index 252bf0c439..89ef84e1e5 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java @@ -165,8 +165,11 @@ public class RecordQueryIndexPlan implements RecordQueryPlanWithNoChildren, @Nonnull private final Supplier comparisonRangesSupplier; - public RecordQueryIndexPlan(@Nonnull final String indexName, @Nonnull final IndexScanParameters scanParameters, final boolean reverse) { - this(indexName, null, scanParameters, IndexFetchMethod.SCAN_AND_FETCH, FetchIndexRecords.PRIMARY_KEY, reverse, false); + @Nonnull + private final KeyValueCursorBase.SerializationMode serializationMode; + + public RecordQueryIndexPlan(@Nonnull final String indexName, @Nonnull final IndexScanParameters scanParameters, final boolean reverse, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { + this(indexName, null, scanParameters, IndexFetchMethod.SCAN_AND_FETCH, FetchIndexRecords.PRIMARY_KEY, reverse, false, serializationMode); } public RecordQueryIndexPlan(@Nonnull final String indexName, @@ -175,8 +178,9 @@ public RecordQueryIndexPlan(@Nonnull final String indexName, @Nonnull final IndexFetchMethod useIndexPrefetch, @Nonnull final FetchIndexRecords fetchIndexRecords, final boolean reverse, - final boolean strictlySorted) { - this(indexName, commonPrimaryKey, scanParameters, useIndexPrefetch, fetchIndexRecords, reverse, strictlySorted, Optional.empty(), new Type.Any(), QueryPlanConstraint.tautology()); + final boolean strictlySorted, + @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { + this(indexName, commonPrimaryKey, scanParameters, useIndexPrefetch, fetchIndexRecords, reverse, strictlySorted, Optional.empty(), new Type.Any(), QueryPlanConstraint.tautology(), serializationMode); } public RecordQueryIndexPlan(@Nonnull final String indexName, @@ -188,8 +192,9 @@ public RecordQueryIndexPlan(@Nonnull final String indexName, final boolean strictlySorted, @Nonnull final MatchCandidate matchCandidate, @Nonnull final Type.Record resultType, - @Nonnull final QueryPlanConstraint constraint) { - this(indexName, commonPrimaryKey, scanParameters, indexFetchMethod, fetchIndexRecords, reverse, strictlySorted, Optional.of(matchCandidate), resultType, constraint); + @Nonnull final QueryPlanConstraint constraint, + @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { + this(indexName, commonPrimaryKey, scanParameters, indexFetchMethod, fetchIndexRecords, reverse, strictlySorted, Optional.of(matchCandidate), resultType, constraint, serializationMode); } protected RecordQueryIndexPlan(@Nonnull final PlanSerializationContext serializationContext, @@ -206,6 +211,19 @@ protected RecordQueryIndexPlan(@Nonnull final PlanSerializationContext serializa QueryPlanConstraint.fromProto(serializationContext, Objects.requireNonNull(recordQueryIndexPlanProto.getConstraint()))); } + protected RecordQueryIndexPlan(@Nonnull final String indexName, + @Nullable final KeyExpression commonPrimaryKey, + @Nonnull final IndexScanParameters scanParameters, + @Nonnull final IndexFetchMethod indexFetchMethod, + @Nonnull final FetchIndexRecords fetchIndexRecords, + final boolean reverse, + final boolean strictlySorted, + @Nonnull final Optional matchCandidateOptional, + @Nonnull final Type resultType, + @Nonnull final QueryPlanConstraint constraint) { + this(indexName, commonPrimaryKey, scanParameters, indexFetchMethod, fetchIndexRecords, reverse, strictlySorted, matchCandidateOptional, resultType, constraint, KeyValueCursorBase.SerializationMode.TO_OLD); + } + @VisibleForTesting public RecordQueryIndexPlan(@Nonnull final String indexName, @Nullable final KeyExpression commonPrimaryKey, @@ -216,7 +234,8 @@ public RecordQueryIndexPlan(@Nonnull final String indexName, final boolean strictlySorted, @Nonnull final Optional matchCandidateOptional, @Nonnull final Type resultType, - @Nonnull final QueryPlanConstraint constraint) { + @Nonnull final QueryPlanConstraint constraint, + @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { this.indexName = indexName; this.commonPrimaryKey = commonPrimaryKey; this.scanParameters = scanParameters; @@ -234,6 +253,7 @@ public RecordQueryIndexPlan(@Nonnull final String indexName, } this.constraint = constraint; this.comparisonRangesSupplier = Suppliers.memoize(this::computeComparisonRanges); + this.serializationMode = serializationMode; } @Nonnull diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/QueryPlanCursorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/QueryPlanCursorTest.java index ba68e638d6..32dd335aa8 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/QueryPlanCursorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/QueryPlanCursorTest.java @@ -154,7 +154,7 @@ private RecordQueryPlan scanPlan() { private RecordQueryPlan indexPlanEquals(String indexName, Object value) { IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, value)), Collections.emptySet())); - return new RecordQueryIndexPlan(indexName, scan, false); + return new RecordQueryIndexPlan(indexName, scan, false, KeyValueCursorBase.SerializationMode.TO_OLD); } private KeyExpression primaryKey() { @@ -190,7 +190,7 @@ public void indexRange() throws Exception { final IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Collections.emptyList(), ImmutableSet.of( new Comparisons.SimpleComparison(Comparisons.Type.GREATER_THAN, 2), new Comparisons.SimpleComparison(Comparisons.Type.LESS_THAN, 4)))); - final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", scan, false); + final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", scan, false, KeyValueCursorBase.SerializationMode.TO_OLD); compareSkipsAndCursors(plan); } @@ -199,7 +199,7 @@ public void reverse() throws Exception { final IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Collections.emptyList(), ImmutableSet.of( new Comparisons.SimpleComparison(Comparisons.Type.GREATER_THAN, 2), new Comparisons.SimpleComparison(Comparisons.Type.LESS_THAN, 4)))); - final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", scan, true); + final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", scan, true, KeyValueCursorBase.SerializationMode.TO_OLD); compareSkipsAndCursors(plan); } @@ -207,7 +207,7 @@ public void reverse() throws Exception { public void in() throws Exception { final IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.ParameterComparison(Comparisons.Type.EQUALS, "in_num")), Collections.emptySet())); final RecordQueryPlan plan = new RecordQueryInValuesJoinPlan( - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", scan, false), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", scan, false, KeyValueCursorBase.SerializationMode.TO_OLD), "in_num", Bindings.Internal.IN, Arrays.asList(2, 4), false, false); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java index 46aec50a58..9cec274cf0 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java @@ -257,7 +257,7 @@ public void testIndexScanOnSecondColumn() throws Exception { new RecordTypeKeyComparison("MySimpleRecord").getComparison() ), Collections.emptySet()); IndexScanParameters scan = IndexScanComparisons.byValue(comparison); - RecordQueryPlan plan = new RecordQueryIndexPlan(index.getName(), scan, false); + RecordQueryPlan plan = new RecordQueryIndexPlan(index.getName(), scan, false, KeyValueCursorBase.SerializationMode.TO_OLD); assertEquals(recs.subList(1, 2), recordStore.executeQuery(query) .map(FDBQueriedRecord::getStoredRecord).asList().join()); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java index bd5d25564b..d0eb11a593 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java @@ -47,6 +47,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.provider.foundationdb.IndexScanBounds; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.provider.foundationdb.MultidimensionalIndexScanBounds; import com.apple.foundationdb.record.provider.foundationdb.query.FDBRecordStoreQueryTestBase; import com.apple.foundationdb.record.query.IndexQueryabilityFilter; @@ -510,7 +511,8 @@ void indexReadTest(final boolean useAsync, final long seed, final int numRecords new HypercubeScanParameters("business", (Long)null, intervalEndInclusive, intervalStartInclusive, null), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -562,7 +564,8 @@ void indexReadWithNullsTest(final boolean useAsync, final long seed, final int n new HypercubeScanParameters("business", (Long)null, intervalEndInclusive, intervalStartInclusive, null), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -617,7 +620,8 @@ void indexReadWithNullsAndMinsTest1(final boolean useAsync) throws Exception { new HypercubeScanParameters("business", (Long)null, 0L, 0L, null), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -672,7 +676,8 @@ void indexReadWithNullsAndMinsTest2(final boolean useAsync) throws Exception { new HypercubeScanParameters("business", Long.MIN_VALUE, 0L, 0L, null), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -722,7 +727,8 @@ void indexReadIsNullTest(final boolean useAsync, final long seed, final int numR new HypercubeScanParameters("business", (Long)null, Long.MIN_VALUE, null, null), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); final Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -788,7 +794,8 @@ void indexReadWithIn(final boolean useAsync, final long seed, final int numRecor new RecordQueryIndexPlan("EventIntervals", new CompositeScanParameters( new MultidimensionalIndexScanBounds(TupleRange.allOf(Tuple.from("business")), andBounds, TupleRange.ALL)), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); final Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -835,7 +842,8 @@ void indexReadsAfterDeletesTest(final boolean useAsync, final long seed, final i new HypercubeScanParameters("business", (Long)null, intervalEndInclusive, intervalStartInclusive, null), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); final Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -871,7 +879,8 @@ void indexSkipScanTest(final boolean useAsync, final long seed, final int numRec new HypercubeScanParameters("business", "private", null, intervalEndInclusive, intervalStartInclusive, null), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); final Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -916,7 +925,8 @@ void continuationTest(final boolean useAsync) throws Exception { new HypercubeScanParameters("business", "private", Long.MIN_VALUE, intervalEndInclusive, intervalStartInclusive, null), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); Set actualResults = getResultsWithContinuations(additionalIndexes, indexPlan, 4); @@ -989,7 +999,8 @@ void coveringIndexScanWithFetchTest(final boolean useAsync, @Nonnull final Strin new HypercubeScanParameters("business", "private", null, intervalEndInclusive, intervalStartInclusive, null), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); final RecordType myMultidimensionalRecord = recordStore.getRecordMetaData().getRecordType("MyMultidimensionalRecord"); final Index index = recordStore.getRecordMetaData().getIndex("EventIntervals"); @@ -1112,7 +1123,8 @@ void indexScan3DTest(final boolean useAsync, final long seed, final int numRecor Long.MIN_VALUE, intervalEndInclusive, intervalStartInclusive, null, epochMean + expirationCutOff, null), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); Set actualResults = getResults(additionalIndex, indexPlan); final QueryComponent filter = @@ -1199,7 +1211,8 @@ void deleteWhereTest(final boolean useAsync, @Nonnull final String storage, fina new RecordQueryIndexPlan("EventIntervals", new CompositeScanParameters( new MultidimensionalIndexScanBounds(TupleRange.allOf(Tuple.from(null, "business")), bounds, TupleRange.ALL)), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); final Set actualResults = getResults(additionalIndexes, indexPlan); Assertions.assertTrue(actualResults.isEmpty()); @@ -1220,7 +1233,8 @@ void unprefixedIndexReadTest(final boolean useAsync, final long seed, final int new HypercubeScanParameters(null, (Long)null, intervalEndInclusive, intervalStartInclusive, null), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java index 2ca83d6c6b..634b46e48a 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java @@ -24,6 +24,7 @@ import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ExecuteProperties; import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.test.Tags; import com.google.common.collect.ImmutableList; @@ -301,7 +302,8 @@ void concurrentReadsAndWrites(@Nonnull final String storage, final boolean store new HypercubeScanParameters("business", (Long)null, null, null, null), - false); + false, + KeyValueCursorBase.SerializationMode.TO_OLD); final var random = new Random(System.currentTimeMillis()); final var expectedMessages = new HashSet(); var writeNum = 0; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreLimitTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreLimitTestBase.java index af5642bfec..c220bb632c 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreLimitTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreLimitTestBase.java @@ -26,6 +26,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.expressions.Query; import com.apple.foundationdb.record.query.expressions.QueryComponent; @@ -75,7 +76,7 @@ void setupSimpleRecordStore() throws Exception { private static RecordQueryPlan indexPlanEquals(String indexName, Object value) { IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, value)), Collections.emptySet())); - return new RecordQueryIndexPlan(indexName, scan, false); + return new RecordQueryIndexPlan(indexName, scan, false, KeyValueCursorBase.SerializationMode.TO_OLD); } private static KeyExpression primaryKey() { @@ -86,7 +87,7 @@ static Stream plans(boolean fail) { RecordQueryPlan scanPlan = new RecordQueryScanPlan(ScanComparisons.EMPTY, false); IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); RecordQueryPlan indexPlan = new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", - fullValueScan, false); + fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD); QueryComponent filter = Query.field("str_value_indexed").equalsValue("odd"); QueryComponent middleFilter = Query.and( Query.field("rec_no").greaterThan(24L), @@ -96,7 +97,7 @@ static Stream plans(boolean fail) { return Stream.of( Arguments.of("full record scan", fail, scanPlan), Arguments.of("simple index scan", fail, indexPlan), - Arguments.of("reverse index scan", fail, new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, true)), + Arguments.of("reverse index scan", fail, new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, true, KeyValueCursorBase.SerializationMode.TO_OLD)), Arguments.of("filter on scan plan", fail, new RecordQueryFilterPlan(scanPlan, filter)), Arguments.of("filter on index plan", fail, new RecordQueryFilterPlan(indexPlan, filter)), Arguments.of("type filter on scan plan", fail, new RecordQueryTypeFilterPlan(scanPlan, Collections.singletonList("MySimpleRecord"))), diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java index 23be321b10..49f9bf825c 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java @@ -44,6 +44,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.provider.foundationdb.SplitHelper; import com.apple.foundationdb.record.provider.foundationdb.cursors.ProbableIntersectionCursor; import com.apple.foundationdb.record.query.RecordQuery; @@ -352,7 +353,7 @@ void testSplitContinuation() { public void testExecuteStateReset(int scanLimit) throws Exception { final IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", - fullValueScan, false); + fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD); ExecuteProperties properties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanLimit).build(); try (FDBRecordContext context = openContext()) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java index c5367a6e87..1ee518b7cc 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java @@ -31,6 +31,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.expressions.Query; @@ -970,8 +971,8 @@ public void intersectionVisitorOnComplexComparisonKey() throws Exception { IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); RecordQueryPlan originalPlan1 = RecordQueryIntersectionPlan.from( - new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false), - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false), + new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), primaryKey("MySimpleRecord")); RecordQueryPlan modifiedPlan1 = RecordQueryPlannerSubstitutionVisitor.applyRegularVisitors(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), originalPlan1, recordStore.getRecordMetaData(), PlannableIndexTypes.DEFAULT, primaryKey("MySimpleRecord")); @@ -979,8 +980,8 @@ public void intersectionVisitorOnComplexComparisonKey() throws Exception { coveringIndexScan(indexScan("MySimpleRecord$str_value_indexed")), coveringIndexScan(indexScan("MySimpleRecord$num_value_3_indexed"))))); RecordQueryPlan originalPlan2 = RecordQueryIntersectionPlan.from( - new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false), - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false), + new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), concat(field("num_value_2"), primaryKey("MySimpleRecord"))); RecordQueryPlan modifiedPlan2 = RecordQueryPlannerSubstitutionVisitor.applyRegularVisitors(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), originalPlan2, recordStore.getRecordMetaData(), PlannableIndexTypes.DEFAULT, primaryKey("MySimpleRecord")); // Visitor should not perform transformation because of comparison key on num_value_unique diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java index c03b44329d..5363662677 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java @@ -40,6 +40,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.expressions.OrComponent; @@ -2346,8 +2347,8 @@ void unionVisitorOnComplexComparisonKey() throws Exception { final IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); RecordQueryPlan originalPlan1 = RecordQueryUnionPlan.from( - new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false), - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false), + new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), primaryKey("MySimpleRecord"), true); RecordQueryPlan modifiedPlan1 = RecordQueryPlannerSubstitutionVisitor.applyRegularVisitors(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), originalPlan1, recordStore.getRecordMetaData(), PlannableIndexTypes.DEFAULT, primaryKey("MySimpleRecord")); @@ -2362,8 +2363,8 @@ void unionVisitorOnComplexComparisonKey() throws Exception { assertMatchesExactly(modifiedPlan1, planMatcher); RecordQueryPlan originalPlan2 = RecordQueryUnionPlan.from( - new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false), - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false), + new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), concat(field("num_value_2"), primaryKey("MySimpleRecord")), true); RecordQueryPlan modifiedPlan2 = RecordQueryPlannerSubstitutionVisitor.applyRegularVisitors(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), originalPlan2, recordStore.getRecordMetaData(), PlannableIndexTypes.DEFAULT, primaryKey("MySimpleRecord")); // Visitor should not perform transformation because of comparison key on num_value_unique diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java index a87dfac728..01a2b696d4 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java @@ -26,6 +26,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInValuesJoinPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; @@ -54,7 +55,7 @@ public class QueryPlanStructuralInstrumentationTest { private RecordQueryPlan indexPlanEquals(String indexName, Object value) { IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, value)), Collections.emptySet())); - return new RecordQueryIndexPlan(indexName, scan, false); + return new RecordQueryIndexPlan(indexName, scan, false, KeyValueCursorBase.SerializationMode.TO_OLD); } private void assertNoIndexes(RecordQueryPlan plan) { @@ -93,7 +94,7 @@ public void in() { final String indexName = "a_field"; final IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.ParameterComparison(Comparisons.Type.EQUALS, "another_field")), Collections.emptySet())); final RecordQueryPlan plan = new RecordQueryInValuesJoinPlan( - new RecordQueryIndexPlan(indexName, scan, false), + new RecordQueryIndexPlan(indexName, scan, false, KeyValueCursorBase.SerializationMode.TO_OLD), "another_field", Bindings.Internal.IN, Arrays.asList(2, 4), diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/matchers/ExpressionMatcherTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/matchers/ExpressionMatcherTest.java index 8a86564bae..7bade28bfe 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/matchers/ExpressionMatcherTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/matchers/ExpressionMatcherTest.java @@ -24,6 +24,7 @@ import com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.expressions.Query; import com.apple.foundationdb.record.query.expressions.QueryComponent; import com.apple.foundationdb.record.query.plan.QueryPlanConstraint; @@ -77,7 +78,7 @@ public class ExpressionMatcherTest { RecordQueryPlanMatchers.indexPlan(), RelationalExpressionMatchers.ofType(RecordQueryPlan.class)); private static final List existingBindables = ImmutableList.of( - new RecordQueryIndexPlan("fake_index", IndexScanComparisons.byValue(), false), + new RecordQueryIndexPlan("fake_index", IndexScanComparisons.byValue(), false, KeyValueCursorBase.SerializationMode.TO_OLD), new RecordQueryScanPlan(ScanComparisons.EMPTY, false)); @Nonnull @@ -122,7 +123,7 @@ public void singleTypeMatcher() { BindingMatcher matcher = RecordQueryPlanMatchers.indexPlan(); final IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); final Reference root = - Reference.plannedOf(new RecordQueryIndexPlan("an_index", fullValueScan, true)); + Reference.plannedOf(new RecordQueryIndexPlan("an_index", fullValueScan, true, KeyValueCursorBase.SerializationMode.TO_OLD)); Optional newBindings = matcher.bindMatches(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), PlannerBindings.empty(), root.get()).findFirst(); // check the bindings are what we expect, and that none of the existing ones were clobbered assertTrue(newBindings.isPresent()); @@ -142,7 +143,7 @@ public void nestedTypeMatchers() { ListMatcher.exactly(QuantifierMatchers.physicalQuantifier(childMatcher1), QuantifierMatchers.physicalQuantifier(childMatcher2))); IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); - RecordQueryIndexPlan child1 = new RecordQueryIndexPlan("an_index", fullValueScan, true); + RecordQueryIndexPlan child1 = new RecordQueryIndexPlan("an_index", fullValueScan, true, KeyValueCursorBase.SerializationMode.TO_OLD); RecordQueryScanPlan child2 = new RecordQueryScanPlan(ScanComparisons.EMPTY, true); // check matches if the children are in the right order @@ -169,7 +170,7 @@ public void matchChildOrder() { QuantifierMatchers.physicalQuantifier(RecordQueryPlanMatchers.scanPlan()))); IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); - RecordQueryIndexPlan child1 = new RecordQueryIndexPlan("an_index", fullValueScan, true); + RecordQueryIndexPlan child1 = new RecordQueryIndexPlan("an_index", fullValueScan, true, KeyValueCursorBase.SerializationMode.TO_OLD); RecordQueryScanPlan child2 = new RecordQueryScanPlan(ScanComparisons.EMPTY, true); RelationalExpression root = RecordQueryUnionPlan.from( // union with arbitrary comparison key child1, child2, EmptyKeyExpression.EMPTY, false); @@ -189,7 +190,7 @@ public void matchChildrenAsReferences() { QuantifierMatchers.physicalQuantifierOverRef(childMatcher2))); IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); - RecordQueryIndexPlan child1 = new RecordQueryIndexPlan("an_index", fullValueScan, true); + RecordQueryIndexPlan child1 = new RecordQueryIndexPlan("an_index", fullValueScan, true, KeyValueCursorBase.SerializationMode.TO_OLD); RecordQueryScanPlan child2 = new RecordQueryScanPlan(ScanComparisons.EMPTY, true); RelationalExpression root = RecordQueryUnionPlan.from( // union with arbitrary comparison key child1, child2, EmptyKeyExpression.EMPTY, false); @@ -234,7 +235,8 @@ public void treeDescentWithMixedBindings() { Type.Record.fromFields(false, ImmutableList.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.INT), Optional.of("field1")), Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("field2")))), - QueryPlanConstraint.tautology())); + QueryPlanConstraint.tautology(), + KeyValueCursorBase.SerializationMode.TO_OLD)); final Quantifier.ForEach quantifier = Quantifier.forEach(baseRef); LogicalFilterExpression filterPlan = new LogicalFilterExpression(Query.and(andBranch1, andBranch2).expand(quantifier, () -> Quantifier.forEach(baseRef)).getPredicates(), diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitorTest.java index 6142e28925..acf5b91cb7 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitorTest.java @@ -38,6 +38,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.expressions.Query; import com.apple.foundationdb.record.query.expressions.QueryComponent; @@ -220,7 +221,7 @@ private static NonnullPair randomIndexDetails(Random r) IndexScanParameters scanParameters = IndexScanComparisons.byValue(comparisons.getLeft(), scanType); String indexName = randomIndexName(r); boolean reverse = r.nextBoolean(); - return NonnullPair.of(new RecordQueryIndexPlan(indexName, scanParameters, reverse), + return NonnullPair.of(new RecordQueryIndexPlan(indexName, scanParameters, reverse, KeyValueCursorBase.SerializationMode.TO_OLD), String.format("%s %s%s%s", indexName, comparisons.getRight(), scanType == IndexScanType.BY_VALUE ? "" : (" " + scanType), reverse ? " REVERSE" : "")); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/planning/RecordQueryPlanEqualityTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/planning/RecordQueryPlanEqualityTest.java index 7040ef9f37..911a8bcbce 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/planning/RecordQueryPlanEqualityTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/planning/RecordQueryPlanEqualityTest.java @@ -24,6 +24,7 @@ import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.expressions.FieldWithComparison; import com.apple.foundationdb.record.query.expressions.Query; @@ -63,7 +64,7 @@ private RecordQueryPlan scanPlan() { private RecordQueryPlan indexPlanEquals(String indexName, Object value) { IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, value)), Collections.emptySet())); - return new RecordQueryIndexPlan(indexName, scan, false); + return new RecordQueryIndexPlan(indexName, scan, false, KeyValueCursorBase.SerializationMode.TO_OLD); } private RecordQueryPlan unionPlan(Object value1, Object value2) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java index e505f40753..6baa30b58c 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java @@ -81,7 +81,7 @@ class RecordQueryIndexPlanWithOverScanTest extends FDBRecordStoreQueryTestBase { private static final RecordQueryPlannerConfiguration plannerConfiguration = RecordQueryPlannerConfiguration.builder() .setIndexScanPreference(QueryPlanner.IndexScanPreference.PREFER_INDEX) .setAttemptFailedInJoinAsOr(true) - .setKeyValueContinuationSerializationMode(KeyValueCursorBase.SerializationMode.TO_NEW) + .setKeyValueCursorContinuationSerializationMode(KeyValueCursorBase.SerializationMode.TO_NEW) .setComplexityThreshold(RecordQueryPlanner.DEFAULT_COMPLEXITY_THRESHOLD) .addValueIndexOverScanNeeded("MySimpleRecord$str_value_indexed") .addValueIndexOverScanNeeded("compoundIndex") diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java index 99443eaee6..c7d58850d1 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java @@ -25,6 +25,7 @@ import com.apple.foundationdb.record.planprotos.PPlanReference; import com.apple.foundationdb.record.planprotos.PValue; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.plan.QueryPlanConstraint; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; @@ -76,7 +77,8 @@ void simpleIndexScanTest() throws Exception { Type.Record.fromFields(false, ImmutableList.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.INT), Optional.of("field1")), Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("field2")))), - QueryPlanConstraint.tautology()); + QueryPlanConstraint.tautology(), + KeyValueCursorBase.SerializationMode.TO_OLD); PlanSerializationContext planSerializationContext = PlanSerializationContext.newForCurrentMode(); final PPlanReference proto = planSerializationContext.toPlanReferenceProto(plan); final byte[] bytes = proto.toByteArray(); @@ -98,7 +100,8 @@ void recordQueryDefaultOnEmptyPlanTest() throws Exception { false, Optional.empty(), Type.primitiveType(Type.TypeCode.INT, true), - QueryPlanConstraint.tautology()); + QueryPlanConstraint.tautology(), + KeyValueCursorBase.SerializationMode.TO_OLD); final var plan = new RecordQueryDefaultOnEmptyPlan(Quantifier .Physical.physicalBuilder().withAlias(CorrelationIdentifier.of("q42")).build( Reference.plannedOf(indexPlan) From d015132bf1250c2806f7e416417e641b63bb544d Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Sun, 15 Jun 2025 20:22:48 -0700 Subject: [PATCH 10/15] save --- .../record/ExecuteProperties.java | 44 +++++++++----- .../foundationdb/KeyValueCursorBase.java | 15 +++-- .../plan/RecordQueryPlannerConfiguration.java | 10 ++-- .../src/main/proto/record_cursor.proto | 5 +- .../foundationdb/QueryPlanCursorTest.java | 58 +++++++++++-------- .../MultidimensionalIndexTestBase.java | 52 ++++++++--------- .../SimpleMultidimensionalIndexTest.java | 40 ++++++++----- .../SlowMultidimensionalIndexTest.java | 43 +++++++++----- .../limits/FDBRecordStoreScanLimitTest.java | 7 ++- .../query/FDBAndQueryToIntersectionTest.java | 15 +++-- .../query/FDBOrQueryToUnionTest.java | 14 +++-- .../query/FDBStreamAggregationTest.java | 1 - ...ueryPlanStructuralInstrumentationTest.java | 39 ++++++++----- .../RecordQueryIndexPlanWithOverScanTest.java | 2 +- .../serialization/PlanSerializationTest.java | 17 ++++-- .../foundationdb/relational/api/Options.java | 5 +- .../recordlayer/RecordLayerIterator.java | 1 + .../recordlayer/query/PlanGenerator.java | 2 + .../recordlayer/query/QueryPlan.java | 1 + .../recordlayer/query/GroupByQueryTests.java | 50 +++++++++------- .../aggregate-index-tests-count.yamsql | 12 ++-- 21 files changed, 259 insertions(+), 174 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/ExecuteProperties.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/ExecuteProperties.java index 825fe56d3e..c3c9d60fe1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/ExecuteProperties.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/ExecuteProperties.java @@ -72,12 +72,13 @@ public class ExecuteProperties { // how record scan limit reached is handled -- false: return early with continuation, true: throw exception private final boolean failOnScanLimitReached; private final boolean isDryRun; + private final boolean kvCursorContSerializeToNew; private final CursorStreamingMode defaultCursorStreamingMode; @SuppressWarnings("java:S107") private ExecuteProperties(int skip, int rowLimit, @Nonnull IsolationLevel isolationLevel, long timeLimit, - @Nonnull ExecuteState state, boolean failOnScanLimitReached, @Nonnull CursorStreamingMode defaultCursorStreamingMode, boolean isDryRun) { + @Nonnull ExecuteState state, boolean failOnScanLimitReached, @Nonnull CursorStreamingMode defaultCursorStreamingMode, boolean isDryRun, boolean kvCursorContSerializeToNew) { this.skip = skip; this.rowLimit = rowLimit; this.isolationLevel = isolationLevel; @@ -86,6 +87,7 @@ private ExecuteProperties(int skip, int rowLimit, @Nonnull IsolationLevel isolat this.failOnScanLimitReached = failOnScanLimitReached; this.defaultCursorStreamingMode = defaultCursorStreamingMode; this.isDryRun = isDryRun; + this.kvCursorContSerializeToNew = kvCursorContSerializeToNew; } @Nonnull @@ -102,7 +104,7 @@ public ExecuteProperties setSkip(final int skip) { if (skip == this.skip) { return this; } - return copy(skip, rowLimit, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + return copy(skip, rowLimit, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } public boolean isDryRun() { @@ -114,9 +116,11 @@ public ExecuteProperties setDryRun(final boolean isDryRun) { if (isDryRun == this.isDryRun) { return this; } - return copy(skip, rowLimit, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + return copy(skip, rowLimit, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } + public boolean isKvCursorContSerializeToNew() {return kvCursorContSerializeToNew;} + /** * Get the limit on the number of rows that will be returned as it would be passed to FDB. @@ -137,7 +141,7 @@ public ExecuteProperties setReturnedRowLimit(final int rowLimit) { if (newLimit == this.rowLimit) { return this; } - return copy(skip, newLimit, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + return copy(skip, newLimit, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } /** @@ -184,7 +188,7 @@ public ExecuteState getState() { */ @Nonnull public ExecuteProperties setState(@Nonnull ExecuteState newState) { - return copy(skip, rowLimit, timeLimit, isolationLevel, newState, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + return copy(skip, rowLimit, timeLimit, isolationLevel, newState, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } /** @@ -193,7 +197,7 @@ public ExecuteProperties setState(@Nonnull ExecuteState newState) { */ @Nonnull public ExecuteProperties clearState() { - return copy(skip, rowLimit, timeLimit, isolationLevel, new ExecuteState(), failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + return copy(skip, rowLimit, timeLimit, isolationLevel, new ExecuteState(), failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } /** @@ -209,7 +213,7 @@ public ExecuteProperties setFailOnScanLimitReached(boolean failOnScanLimitReache if (failOnScanLimitReached == this.failOnScanLimitReached) { return this; } - return copy(skip, rowLimit, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + return copy(skip, rowLimit, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } @Nonnull @@ -217,7 +221,7 @@ public ExecuteProperties clearReturnedRowLimit() { if (getReturnedRowLimit() == ReadTransaction.ROW_LIMIT_UNLIMITED) { return this; } - return copy(skip, ReadTransaction.ROW_LIMIT_UNLIMITED, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + return copy(skip, ReadTransaction.ROW_LIMIT_UNLIMITED, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } /** @@ -229,7 +233,7 @@ public ExecuteProperties clearRowAndTimeLimits() { if (getTimeLimit() == UNLIMITED_TIME && getReturnedRowLimit() == ReadTransaction.ROW_LIMIT_UNLIMITED) { return this; } - return copy(skip, ReadTransaction.ROW_LIMIT_UNLIMITED, UNLIMITED_TIME, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + return copy(skip, ReadTransaction.ROW_LIMIT_UNLIMITED, UNLIMITED_TIME, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } /** @@ -241,7 +245,7 @@ public ExecuteProperties clearSkipAndLimit() { if (skip == 0 && rowLimit == ReadTransaction.ROW_LIMIT_UNLIMITED) { return this; } - return copy(0, ReadTransaction.ROW_LIMIT_UNLIMITED, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + return copy(0, ReadTransaction.ROW_LIMIT_UNLIMITED, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } /** @@ -254,7 +258,7 @@ public ExecuteProperties clearSkipAndAdjustLimit() { return this; } return copy(0, rowLimit == ReadTransaction.ROW_LIMIT_UNLIMITED ? ReadTransaction.ROW_LIMIT_UNLIMITED : rowLimit + skip, - timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } /** @@ -305,7 +309,7 @@ public ExecuteProperties setDefaultCursorStreamingMode(CursorStreamingMode defau if (defaultCursorStreamingMode == this.defaultCursorStreamingMode) { return this; } - return copy(skip, rowLimit, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + return copy(skip, rowLimit, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } /** @@ -315,7 +319,7 @@ public ExecuteProperties setDefaultCursorStreamingMode(CursorStreamingMode defau */ @Nonnull public ExecuteProperties resetState() { - return copy(skip, rowLimit, timeLimit, isolationLevel, state.reset(), failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + return copy(skip, rowLimit, timeLimit, isolationLevel, state.reset(), failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } /** @@ -333,8 +337,8 @@ public ExecuteProperties resetState() { @SuppressWarnings("java:S107") @Nonnull protected ExecuteProperties copy(int skip, int rowLimit, long timeLimit, @Nonnull IsolationLevel isolationLevel, - @Nonnull ExecuteState state, boolean failOnScanLimitReached, CursorStreamingMode defaultCursorStreamingMode, boolean isDryRun) { - return new ExecuteProperties(skip, rowLimit, isolationLevel, timeLimit, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + @Nonnull ExecuteState state, boolean failOnScanLimitReached, CursorStreamingMode defaultCursorStreamingMode, boolean isDryRun, boolean kvCursorContSerializeToNew) { + return new ExecuteProperties(skip, rowLimit, isolationLevel, timeLimit, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } @Nonnull @@ -408,6 +412,7 @@ public static class Builder { private ExecuteState executeState = null; private boolean failOnScanLimitReached = false; private boolean isDryRun = false; + private boolean kvCursorContSerializeToNew = false; private CursorStreamingMode defaultCursorStreamingMode = CursorStreamingMode.ITERATOR; private Builder() { @@ -422,6 +427,7 @@ private Builder(ExecuteProperties executeProperties) { this.failOnScanLimitReached = executeProperties.failOnScanLimitReached; this.defaultCursorStreamingMode = executeProperties.defaultCursorStreamingMode; this.isDryRun = executeProperties.isDryRun; + this.kvCursorContSerializeToNew = executeProperties.kvCursorContSerializeToNew; } @Nonnull @@ -455,6 +461,12 @@ public Builder setDryRun(boolean isDryRun) { return this; } + @Nonnull + public Builder setKvCursorContSerializeToNew(boolean kvCursorContSerializeToNew) { + this.kvCursorContSerializeToNew = kvCursorContSerializeToNew; + return this; + } + @Nonnull public Builder setReturnedRowLimit(int rowLimit) { this.rowLimit = validateAndNormalizeRowLimit(rowLimit); @@ -607,7 +619,7 @@ public ExecuteProperties build() { } else { state = new ExecuteState(RecordScanLimiterFactory.enforce(scannedRecordsLimit), ByteScanLimiterFactory.enforce(scannedBytesLimit)); } - return new ExecuteProperties(skip, rowLimit, isolationLevel, timeLimit, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun); + return new ExecuteProperties(skip, rowLimit, isolationLevel, timeLimit, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java index 31aae73759..e24e016148 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java @@ -228,11 +228,11 @@ public static byte[] fromRawBytes(@Nonnull byte[] rawBytes, SerializationMode se private RecordCursorProto.KeyValueCursorContinuation toProto() { RecordCursorProto.KeyValueCursorContinuation.Builder builder = RecordCursorProto.KeyValueCursorContinuation.newBuilder(); if (lastKey == null) { - builder.setIsEnd(true); + builder.setIsEnd(true).setPrefixLength(prefixLength); } else { ByteString base = ZeroCopyByteString.wrap(lastKey); builder.setContinuation(base.substring(prefixLength, lastKey.length)) - .setIsEnd(false); + .setIsEnd(false).setPrefixLength(prefixLength); } return builder.build(); } @@ -278,8 +278,7 @@ public abstract static class Builder> { private StreamingMode streamingMode; private KeySelector begin; private KeySelector end; - // default to be old now - protected SerializationMode serializationMode = SerializationMode.TO_OLD; + protected SerializationMode serializationMode; protected Builder(@Nonnull Subspace subspace) { this.subspace = subspace; @@ -319,6 +318,8 @@ protected void prepare() { prefixLength = calculatePrefixLength(); reverse = scanProperties.isReverse(); + serializationMode = scanProperties.getExecuteProperties().isKvCursorContSerializeToNew() ? SerializationMode.TO_NEW : SerializationMode.TO_OLD; + if (continuation != null) { byte[] realContinuation; if (serializationMode == SerializationMode.TO_OLD) { @@ -326,7 +327,11 @@ protected void prepare() { } else { try { RecordCursorProto.KeyValueCursorContinuation keyValueCursorContinuation = RecordCursorProto.KeyValueCursorContinuation.parseFrom(continuation); - realContinuation = keyValueCursorContinuation.getContinuation().toByteArray(); + if (keyValueCursorContinuation.hasIsEnd()) { + realContinuation = keyValueCursorContinuation.getContinuation().toByteArray(); + } else { + realContinuation = continuation; + } } catch (InvalidProtocolBufferException ex) { System.out.println("InvalidProtocolBufferException hit"); realContinuation = continuation; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java index fbb1439496..17b79658b9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java @@ -441,13 +441,13 @@ public Builder() { } @Nonnull - public Builder setKeyValueCursorContinuationSerializationMode(@Nonnull KeyValueCursorBase.SerializationMode serializationMode) { - if (serializationMode == KeyValueCursorBase.SerializationMode.TO_OLD) { - protoBuilder.setKeyValueCursorContinuationSerializationMode(RecordPlannerConfigurationProto.PlannerConfiguration.KeyValueCursorContinuationSerializationMode.TO_OLD); - updateFlags(false, KEYVALUE_CURSOR_CONTINUATION_SERIALIZATION_MODE); - } else { + public Builder setKeyValueCursorContinuationSerializationMode(boolean toNew) { + if (toNew) { protoBuilder.setKeyValueCursorContinuationSerializationMode(RecordPlannerConfigurationProto.PlannerConfiguration.KeyValueCursorContinuationSerializationMode.TO_NEW); updateFlags(true, KEYVALUE_CURSOR_CONTINUATION_SERIALIZATION_MODE); + } else { + protoBuilder.setKeyValueCursorContinuationSerializationMode(RecordPlannerConfigurationProto.PlannerConfiguration.KeyValueCursorContinuationSerializationMode.TO_OLD); + updateFlags(false, KEYVALUE_CURSOR_CONTINUATION_SERIALIZATION_MODE); } return this; } diff --git a/fdb-record-layer-core/src/main/proto/record_cursor.proto b/fdb-record-layer-core/src/main/proto/record_cursor.proto index 632390adcd..c806963b18 100644 --- a/fdb-record-layer-core/src/main/proto/record_cursor.proto +++ b/fdb-record-layer-core/src/main/proto/record_cursor.proto @@ -167,6 +167,7 @@ message RangeCursorContinuation { } message KeyValueCursorContinuation { - optional bytes continuation = 1; - optional bool isEnd = 2; + optional int32 prefixLength = 1; + optional bytes continuation = 2; + optional bool isEnd = 3; } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/QueryPlanCursorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/QueryPlanCursorTest.java index 32dd335aa8..0a24a4d3be 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/QueryPlanCursorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/QueryPlanCursorTest.java @@ -46,6 +46,8 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -151,10 +153,10 @@ private RecordQueryPlan scanPlan() { return new RecordQueryScanPlan(ScanComparisons.EMPTY, false); } - private RecordQueryPlan indexPlanEquals(String indexName, Object value) { + private RecordQueryPlan indexPlanEquals(String indexName, Object value, KeyValueCursorBase.SerializationMode serializationMode) { IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, value)), Collections.emptySet())); - return new RecordQueryIndexPlan(indexName, scan, false, KeyValueCursorBase.SerializationMode.TO_OLD); + return new RecordQueryIndexPlan(indexName, scan, false, serializationMode); } private KeyExpression primaryKey() { @@ -179,9 +181,10 @@ public void indexlessFilter() throws Exception { compareSkipsAndCursors(plan); } - @Test - public void indexed() throws Exception { - final RecordQueryPlan plan = indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2); + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void indexed(KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + final RecordQueryPlan plan = indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2, serializationMode); compareSkipsAndCursors(plan); } @@ -214,51 +217,55 @@ public void in() throws Exception { compareSkipsAndCursors(plan); } - @Test - public void union() throws Exception { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void union(KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordQueryPlan plan = RecordQueryUnionPlan.from( - indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2), - indexPlanEquals("MySimpleRecord$num_value_3_indexed", 4), + indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2, serializationMode), + indexPlanEquals("MySimpleRecord$num_value_3_indexed", 4, serializationMode), primaryKey(), false); compareSkipsAndCursors(plan); } - @Test - public void unionOneSideAtATime() throws Exception { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void unionOneSideAtATime(KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordQueryPlan plan = RecordQueryUnionPlan.from( - indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2), - indexPlanEquals("MySimpleRecord$num_value_3_indexed", 4), + indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2, serializationMode), + indexPlanEquals("MySimpleRecord$num_value_3_indexed", 4, serializationMode), Key.Expressions.concat(Key.Expressions.field("num_value_3_indexed"), primaryKey()), true); compareSkipsAndCursors(plan); } - @Test - public void intersection() throws Exception { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void intersection(KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordQueryPlan plan = RecordQueryIntersectionPlan.from( - indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2), - indexPlanEquals("MySimpleRecord$str_value_indexed", "even"), + indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2, serializationMode), + indexPlanEquals("MySimpleRecord$str_value_indexed", "even", serializationMode), primaryKey()); compareSkipsAndCursors(plan); } - @Test - public void filter() throws Exception { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void filter(KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordQueryPlan plan = new RecordQueryFilterPlan( - indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2), + indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2, serializationMode), Query.field("str_value_indexed").equalsValue("even") ); compareSkipsAndCursors(plan); } - private void filterKeyCount(int amount) throws Exception { + private void filterKeyCount(int amount, KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final QueryComponent filter = Query.field("str_value_indexed").equalsValue("even"); try (FDBRecordContext context = openContext()) { openSimpleRecordStore(context); recordStore.getTimer().reset(); - final RecordQueryPlan plan = indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2); + final RecordQueryPlan plan = indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2, serializationMode); byte[] continuation = null; int unfilteredCount = 0; @@ -300,10 +307,11 @@ private void filterKeyCount(int amount) throws Exception { } } - @Test - public void filterKeyCount() throws Exception { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void filterKeyCount(KeyValueCursorBase.SerializationMode serializationMode) throws Exception { for (int amount : amounts) { - filterKeyCount(amount); + filterKeyCount(amount, serializationMode); } } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java index d0eb11a593..99edc250d4 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java @@ -497,7 +497,7 @@ void basicReadWithNullsTest(final boolean useAsync, @Nonnull final String storag } void indexReadTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -512,7 +512,7 @@ void indexReadTest(final boolean useAsync, final long seed, final int numRecords (Long)null, intervalEndInclusive, intervalStartInclusive, null), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -550,7 +550,7 @@ void indexReadTest(final boolean useAsync, final long seed, final int numRecords } void indexReadWithNullsTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -565,7 +565,7 @@ void indexReadWithNullsTest(final boolean useAsync, final long seed, final int n (Long)null, intervalEndInclusive, intervalStartInclusive, null), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -608,7 +608,7 @@ void indexReadWithNullsTest(final boolean useAsync, final long seed, final int n Assertions.assertEquals(expectedResults, actualResults); } - void indexReadWithNullsAndMinsTest1(final boolean useAsync) throws Exception { + void indexReadWithNullsAndMinsTest1(final boolean useAsync, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -621,7 +621,7 @@ void indexReadWithNullsAndMinsTest1(final boolean useAsync) throws Exception { (Long)null, 0L, 0L, null), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -664,7 +664,7 @@ void indexReadWithNullsAndMinsTest1(final boolean useAsync) throws Exception { Assertions.assertEquals(expectedResults, actualResults); } - void indexReadWithNullsAndMinsTest2(final boolean useAsync) throws Exception { + void indexReadWithNullsAndMinsTest2(final boolean useAsync, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -677,7 +677,7 @@ void indexReadWithNullsAndMinsTest2(final boolean useAsync) throws Exception { Long.MIN_VALUE, 0L, 0L, null), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -715,7 +715,7 @@ void indexReadWithNullsAndMinsTest2(final boolean useAsync) throws Exception { } void indexReadIsNullTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -728,7 +728,7 @@ void indexReadIsNullTest(final boolean useAsync, final long seed, final int numR (Long)null, Long.MIN_VALUE, null, null), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); final Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -755,7 +755,7 @@ void indexReadIsNullTest(final boolean useAsync, final long seed, final int numR Assertions.assertTrue(plan2.hasIndexScan("calendarNameStartEpoch")); } - void indexReadWithIn(final boolean useAsync, final long seed, final int numRecords, final int numIns) throws Exception { + void indexReadWithIn(final boolean useAsync, final long seed, final int numRecords, final int numIns, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -795,7 +795,7 @@ void indexReadWithIn(final boolean useAsync, final long seed, final int numRecor new CompositeScanParameters( new MultidimensionalIndexScanBounds(TupleRange.allOf(Tuple.from("business")), andBounds, TupleRange.ALL)), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); final Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -827,7 +827,7 @@ void indexReadWithIn(final boolean useAsync, final long seed, final int numRecor void indexReadsAfterDeletesTest(final boolean useAsync, final long seed, final int numRecords, final int numDeletes, @Nonnull final String storage, final boolean storeHilbertValues, - final boolean useNodeSlotIndex) throws Exception { + final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -843,7 +843,7 @@ void indexReadsAfterDeletesTest(final boolean useAsync, final long seed, final i (Long)null, intervalEndInclusive, intervalStartInclusive, null), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); final Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -864,7 +864,7 @@ void indexReadsAfterDeletesTest(final boolean useAsync, final long seed, final i } void indexSkipScanTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -880,7 +880,7 @@ void indexSkipScanTest(final boolean useAsync, final long seed, final int numRec null, intervalEndInclusive, intervalStartInclusive, null), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); final Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -910,7 +910,7 @@ void indexSkipScanTest(final boolean useAsync, final long seed, final int numRec Assertions.assertTrue(plan2.hasRecordScan()); } - void continuationTest(final boolean useAsync) throws Exception { + void continuationTest(final boolean useAsync, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -926,7 +926,7 @@ void continuationTest(final boolean useAsync) throws Exception { Long.MIN_VALUE, intervalEndInclusive, intervalStartInclusive, null), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); Set actualResults = getResultsWithContinuations(additionalIndexes, indexPlan, 4); @@ -984,7 +984,7 @@ private Set getResultsWithContinuations(@Nonnull final RecordMetaDataHo } void coveringIndexScanWithFetchTest(final boolean useAsync, @Nonnull final String storage, final boolean storeHilbertValues, - final boolean useNodeSlotIndex) throws Exception { + final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -1000,7 +1000,7 @@ void coveringIndexScanWithFetchTest(final boolean useAsync, @Nonnull final Strin null, intervalEndInclusive, intervalStartInclusive, null), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); final RecordType myMultidimensionalRecord = recordStore.getRecordMetaData().getRecordType("MyMultidimensionalRecord"); final Index index = recordStore.getRecordMetaData().getIndex("EventIntervals"); @@ -1104,7 +1104,7 @@ void coveringIndexReadTest(final boolean useAsync, final long seed, final int nu } void indexScan3DTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordMetaDataHook additionalIndex = metaDataBuilder -> metaDataBuilder.addIndex("MyMultidimensionalRecord", new Index("EventIntervals3D", DimensionsKeyExpression.of(field("calendar_name"), @@ -1124,7 +1124,7 @@ void indexScan3DTest(final boolean useAsync, final long seed, final int numRecor intervalStartInclusive, null, epochMean + expirationCutOff, null), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); Set actualResults = getResults(additionalIndex, indexPlan); final QueryComponent filter = @@ -1180,7 +1180,7 @@ void wrongDimensionTypes(final boolean useAsync) { Assertions.assertEquals(KeyExpression.InvalidExpressionException.class, cause.getClass()); } - void deleteWhereTest(final boolean useAsync, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { + void deleteWhereTest(final boolean useAsync, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { metaDataBuilder.addIndex("MyMultidimensionalRecord", @@ -1212,14 +1212,14 @@ void deleteWhereTest(final boolean useAsync, @Nonnull final String storage, fina new CompositeScanParameters( new MultidimensionalIndexScanBounds(TupleRange.allOf(Tuple.from(null, "business")), bounds, TupleRange.ALL)), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); final Set actualResults = getResults(additionalIndexes, indexPlan); Assertions.assertTrue(actualResults.isEmpty()); } void unprefixedIndexReadTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -1234,7 +1234,7 @@ void unprefixedIndexReadTest(final boolean useAsync, final long seed, final int (Long)null, intervalEndInclusive, intervalStartInclusive, null), false, - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java index f203aa1d04..23c428edbd 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.record.provider.foundationdb.indexes; +import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -178,35 +179,40 @@ void basicReadWithNullsTest(@Nonnull final String storage, final boolean storeHi @MethodSource("argumentsForBasicReads") void deleteWhereTest(@Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.deleteWhereTest(false, storage, storeHilbertValues, useNodeSlotIndex); + super.deleteWhereTest(false, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.deleteWhereTest(false, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @MethodSource("argumentsForBasicReads") void coveringIndexScanWithFetchTest(@Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.coveringIndexScanWithFetchTest(false, storage, storeHilbertValues, useNodeSlotIndex); + super.coveringIndexScanWithFetchTest(false, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.coveringIndexScanWithFetchTest(false, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void indexReadTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); + super.indexReadTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void indexReadWithNullsTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadWithNullsTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); + super.indexReadWithNullsTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadWithNullsTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void indexReadIsNullTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadIsNullTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); + super.indexReadIsNullTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadIsNullTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @@ -220,14 +226,16 @@ void coveringIndexReadTest(final long seed, final int numRecords, @Nonnull final @MethodSource("argumentsForIndexReads") void indexScan3DTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexScan3DTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); + super.indexScan3DTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexScan3DTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void unprefixedIndexReadTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.unprefixedIndexReadTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); + super.unprefixedIndexReadTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.unprefixedIndexReadTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @@ -255,7 +263,8 @@ void unprefixedSuffixedIndexReadWithResidualsTest(final long seed, final int num @MethodSource("argumentsForIndexReads") void indexSkipScanTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexSkipScanTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); + super.indexSkipScanTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexSkipScanTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @@ -263,7 +272,8 @@ void indexSkipScanTest(final long seed, final int numRecords, @Nonnull final Str void indexReadsAfterDeletesTest(final long seed, final int numRecords, final int numDeletes, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadsAfterDeletesTest(false, seed, numRecords, numDeletes, storage, storeHilbertValues, useNodeSlotIndex); + super.indexReadsAfterDeletesTest(false, seed, numRecords, numDeletes, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadsAfterDeletesTest(false, seed, numRecords, numDeletes, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @@ -275,7 +285,8 @@ void indexReadWithDuplicatesTest(final int numRecords, @Nonnull final String sto @Test void continuationTest() throws Exception { - super.continuationTest(false); + super.continuationTest(false, KeyValueCursorBase.SerializationMode.TO_OLD); + super.continuationTest(false, KeyValueCursorBase.SerializationMode.TO_NEW); } @Test @@ -285,17 +296,20 @@ void wrongDimensionTypes() { @Test void indexReadWithNullsAndMinsTest1() throws Exception { - super.indexReadWithNullsAndMinsTest1(false); + super.indexReadWithNullsAndMinsTest1(false, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadWithNullsAndMinsTest1(false, KeyValueCursorBase.SerializationMode.TO_NEW); } @Test void indexReadWithNullsAndMinsTest2() throws Exception { - super.indexReadWithNullsAndMinsTest2(false); + super.indexReadWithNullsAndMinsTest2(false, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadWithNullsAndMinsTest2(false, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @MethodSource("argumentsForIndexReadWithIn") void indexReadWithIn(final long seed, final int numRecords, final int numIns) throws Exception { - super.indexReadWithIn(false, seed, numRecords, numIns); + super.indexReadWithIn(false, seed, numRecords, numIns, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadWithIn(false, seed, numRecords, numIns, KeyValueCursorBase.SerializationMode.TO_NEW); } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java index 634b46e48a..10a021bce6 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java @@ -26,6 +26,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; import com.apple.test.Tags; import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; @@ -109,7 +110,7 @@ static Stream argumentsForIndexReadsAfterDeletes() { Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_NODE.toString(), false, true), Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_NODE.toString(), true, false), Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_NODE.toString(), true, true) - )); + )); } static Stream argumentsForIndexReadsWithDuplicates() { @@ -171,35 +172,40 @@ void basicReadWithNulls(@Nonnull final String storage, final boolean storeHilber @ParameterizedTest @MethodSource("argumentsForBasicReads") void deleteWhereTest(@Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.deleteWhereTest(true, storage, storeHilbertValues, useNodeSlotIndex); + super.deleteWhereTest(true, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.deleteWhereTest(true, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @MethodSource("argumentsForBasicReads") void coveringIndexScanWithFetchTest(@Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.coveringIndexScanWithFetchTest(true, storage, storeHilbertValues, useNodeSlotIndex); + super.coveringIndexScanWithFetchTest(true, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.coveringIndexScanWithFetchTest(true, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void indexReadTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); + super.indexReadTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void indexReadWithNullsTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadWithNullsTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); + super.indexReadWithNullsTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadWithNullsTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void indexReadIsNullTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadIsNullTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); + super.indexReadIsNullTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadIsNullTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @@ -213,14 +219,16 @@ void coveringIndexReadTest(final long seed, final int numRecords, @Nonnull final @MethodSource("argumentsForIndexReads") void indexScan3DTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexScan3DTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); + super.indexScan3DTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexScan3DTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void unprefixedIndexReadTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.unprefixedIndexReadTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); + super.unprefixedIndexReadTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.unprefixedIndexReadTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @@ -248,7 +256,8 @@ void unprefixedSuffixedIndexReadWithResidualsTest(final long seed, final int num @MethodSource("argumentsForIndexReads") void indexSkipScanTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexSkipScanTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); + super.indexSkipScanTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexSkipScanTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @@ -256,7 +265,9 @@ void indexSkipScanTest(final long seed, final int numRecords, @Nonnull final Str void indexReadsAfterDeletesTest(final long seed, final int numRecords, final int numDeletes, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadsAfterDeletesTest(true, seed, numRecords, numDeletes, storage, storeHilbertValues, useNodeSlotIndex); + super.indexReadsAfterDeletesTest(true, seed, numRecords, numDeletes, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadsAfterDeletesTest(true, seed, numRecords, numDeletes, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + } @ParameterizedTest @@ -268,7 +279,8 @@ void indexReadWithDuplicatesTest(final int numRecords, @Nonnull final String sto @Test void continuationTest() throws Exception { - super.continuationTest(true); + super.continuationTest(true, KeyValueCursorBase.SerializationMode.TO_OLD); + super.continuationTest(true, KeyValueCursorBase.SerializationMode.TO_NEW); } @Test @@ -278,18 +290,21 @@ void wrongDimensionTypes() { @Test void indexReadWithNullsAndMinsTest1() throws Exception { - super.indexReadWithNullsAndMinsTest1(true); + super.indexReadWithNullsAndMinsTest1(true, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadWithNullsAndMinsTest1(true, KeyValueCursorBase.SerializationMode.TO_NEW); } @Test void indexReadWithNullsAndMinsTest2() throws Exception { - super.indexReadWithNullsAndMinsTest2(true); + super.indexReadWithNullsAndMinsTest2(true, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadWithNullsAndMinsTest2(true, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest @MethodSource("argumentsForIndexReadWithIn") void indexReadWithIn(final long seed, final int numRecords, final int numIns) throws Exception { - super.indexReadWithIn(false, seed, numRecords, numIns); + super.indexReadWithIn(false, seed, numRecords, numIns, KeyValueCursorBase.SerializationMode.TO_OLD); + super.indexReadWithIn(false, seed, numRecords, numIns, KeyValueCursorBase.SerializationMode.TO_NEW); } @ParameterizedTest diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java index 49f9bf825c..ab40c12b35 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java @@ -351,9 +351,14 @@ void testSplitContinuation() { @ParameterizedTest @ValueSource(ints = {2, 5, 10, 20}) // for this test, the scan limit must divide 100 public void testExecuteStateReset(int scanLimit) throws Exception { + executeStateReset(scanLimit, KeyValueCursorBase.SerializationMode.TO_OLD); + executeStateReset(scanLimit, KeyValueCursorBase.SerializationMode.TO_NEW); + } + + private void executeStateReset(int scanLimit, @Nonnull KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", - fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD); + fullValueScan, false, serializationMode); ExecuteProperties properties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanLimit).build(); try (FDBRecordContext context = openContext()) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java index 1ee518b7cc..e6f7bf4a4a 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java @@ -54,7 +54,9 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import javax.annotation.Nonnull; import java.util.Arrays; import java.util.Objects; @@ -964,15 +966,16 @@ public void sortedIntersectionBounded(boolean shouldDeferFetch) throws Exception * This sort of plan is never produced by the {@link com.apple.foundationdb.record.query.plan.RecordQueryPlanner}, * so we have to test the visitor directly. */ - @Test - public void intersectionVisitorOnComplexComparisonKey() throws Exception { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void intersectionVisitorOnComplexComparisonKey(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { complexQuerySetup(null); IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); RecordQueryPlan originalPlan1 = RecordQueryIntersectionPlan.from( - new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), + new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, serializationMode), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, serializationMode), primaryKey("MySimpleRecord")); RecordQueryPlan modifiedPlan1 = RecordQueryPlannerSubstitutionVisitor.applyRegularVisitors(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), originalPlan1, recordStore.getRecordMetaData(), PlannableIndexTypes.DEFAULT, primaryKey("MySimpleRecord")); @@ -980,8 +983,8 @@ public void intersectionVisitorOnComplexComparisonKey() throws Exception { coveringIndexScan(indexScan("MySimpleRecord$str_value_indexed")), coveringIndexScan(indexScan("MySimpleRecord$num_value_3_indexed"))))); RecordQueryPlan originalPlan2 = RecordQueryIntersectionPlan.from( - new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), + new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, serializationMode), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, serializationMode), concat(field("num_value_2"), primaryKey("MySimpleRecord"))); RecordQueryPlan modifiedPlan2 = RecordQueryPlannerSubstitutionVisitor.applyRegularVisitors(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), originalPlan2, recordStore.getRecordMetaData(), PlannableIndexTypes.DEFAULT, primaryKey("MySimpleRecord")); // Visitor should not perform transformation because of comparison key on num_value_unique diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java index 5363662677..779d3a175f 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java @@ -74,6 +74,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import javax.annotation.Nonnull; @@ -2340,15 +2341,16 @@ void testOrQueryNoIndex(OrQueryParams orQueryParams) throws Exception { * This sort of plan is never produced by the {@link com.apple.foundationdb.record.query.plan.RecordQueryPlanner}, * so we have to test the visitor directly. */ - @Test - void unionVisitorOnComplexComparisonKey() throws Exception { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + void unionVisitorOnComplexComparisonKey(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { complexQuerySetup(null); final IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); RecordQueryPlan originalPlan1 = RecordQueryUnionPlan.from( - new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), + new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, serializationMode), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, serializationMode), primaryKey("MySimpleRecord"), true); RecordQueryPlan modifiedPlan1 = RecordQueryPlannerSubstitutionVisitor.applyRegularVisitors(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), originalPlan1, recordStore.getRecordMetaData(), PlannableIndexTypes.DEFAULT, primaryKey("MySimpleRecord")); @@ -2363,8 +2365,8 @@ void unionVisitorOnComplexComparisonKey() throws Exception { assertMatchesExactly(modifiedPlan1, planMatcher); RecordQueryPlan originalPlan2 = RecordQueryUnionPlan.from( - new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD), + new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, serializationMode), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, serializationMode), concat(field("num_value_2"), primaryKey("MySimpleRecord")), true); RecordQueryPlan modifiedPlan2 = RecordQueryPlannerSubstitutionVisitor.applyRegularVisitors(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), originalPlan2, recordStore.getRecordMetaData(), PlannableIndexTypes.DEFAULT, primaryKey("MySimpleRecord")); // Visitor should not perform transformation because of comparison key on num_value_unique diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java index d42052619a..d0c5782248 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBStreamAggregationTest.java @@ -60,7 +60,6 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java index 01a2b696d4..abf7f307b3 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java @@ -36,7 +36,10 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryUnionPlan; import com.google.common.collect.Lists; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -52,10 +55,10 @@ public class QueryPlanStructuralInstrumentationTest { private static int VALUE = 4; - private RecordQueryPlan indexPlanEquals(String indexName, Object value) { + private RecordQueryPlan indexPlanEquals(String indexName, Object value, KeyValueCursorBase.SerializationMode serializationMode) { IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, value)), Collections.emptySet())); - return new RecordQueryIndexPlan(indexName, scan, false, KeyValueCursorBase.SerializationMode.TO_OLD); + return new RecordQueryIndexPlan(indexName, scan, false, serializationMode); } private void assertNoIndexes(RecordQueryPlan plan) { @@ -68,11 +71,12 @@ private void assertUsesIndexes(RecordQueryPlan plan, Collection indexes) assertTrue(used.containsAll(indexes)); } - @Test - public void indexPlan() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void indexPlan(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { final String indexName = "an_index"; StoreTimer timer = new FDBStoreTimer(); - RecordQueryPlan plan = indexPlanEquals(indexName, VALUE); + RecordQueryPlan plan = indexPlanEquals(indexName, VALUE, serializationMode); plan.logPlanStructure(timer); assertUsesIndexes(plan, Lists.newArrayList(indexName)); @@ -89,12 +93,13 @@ public void fullScan() { assertEquals(timer.getCount(FDBStoreTimer.Counts.PLAN_SCAN), 1); } - @Test - public void in() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void in(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { final String indexName = "a_field"; final IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.ParameterComparison(Comparisons.Type.EQUALS, "another_field")), Collections.emptySet())); final RecordQueryPlan plan = new RecordQueryInValuesJoinPlan( - new RecordQueryIndexPlan(indexName, scan, false, KeyValueCursorBase.SerializationMode.TO_OLD), + new RecordQueryIndexPlan(indexName, scan, false, serializationMode), "another_field", Bindings.Internal.IN, Arrays.asList(2, 4), @@ -108,11 +113,12 @@ public void in() { assertEquals(timer.getCount(FDBStoreTimer.Counts.PLAN_INDEX), 1); } - @Test - public void unionSameIndex() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void unionSameIndex(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { final RecordQueryPlan plan = RecordQueryUnionPlan.from( - indexPlanEquals("index_1", 2), - indexPlanEquals("index_1", 4), + indexPlanEquals("index_1", 2, serializationMode), + indexPlanEquals("index_1", 4, serializationMode), EmptyKeyExpression.EMPTY, false); assertUsesIndexes(plan, Lists.newArrayList("index_1")); @@ -123,11 +129,12 @@ public void unionSameIndex() { assertEquals(timer.getCount(FDBStoreTimer.Counts.PLAN_INDEX), 2); } - @Test - public void unionDifferentIndex() { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + public void unionDifferentIndex(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { final RecordQueryPlan plan = RecordQueryUnionPlan.from( - indexPlanEquals("index_1", 2), - indexPlanEquals("index_2", 4), + indexPlanEquals("index_1", 2, serializationMode), + indexPlanEquals("index_2", 4, serializationMode), EmptyKeyExpression.EMPTY, false); assertUsesIndexes(plan, Lists.newArrayList("index_1", "index_2")); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java index 6baa30b58c..5f56a10a5e 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java @@ -81,7 +81,7 @@ class RecordQueryIndexPlanWithOverScanTest extends FDBRecordStoreQueryTestBase { private static final RecordQueryPlannerConfiguration plannerConfiguration = RecordQueryPlannerConfiguration.builder() .setIndexScanPreference(QueryPlanner.IndexScanPreference.PREFER_INDEX) .setAttemptFailedInJoinAsOr(true) - .setKeyValueCursorContinuationSerializationMode(KeyValueCursorBase.SerializationMode.TO_NEW) + .setKeyValueCursorContinuationSerializationMode(true) .setComplexityThreshold(RecordQueryPlanner.DEFAULT_COMPLEXITY_THRESHOLD) .addValueIndexOverScanNeeded("MySimpleRecord$str_value_indexed") .addValueIndexOverScanNeeded("compoundIndex") diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java index c7d58850d1..9676e9ff06 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java @@ -43,7 +43,10 @@ import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import javax.annotation.Nonnull; import java.util.Optional; /** @@ -64,8 +67,9 @@ void simpleFieldValueTest() throws Exception { Assertions.assertEquals(fieldValue, parsedValue); } - @Test - void simpleIndexScanTest() throws Exception { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + void simpleIndexScanTest(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordQueryIndexPlan plan = new RecordQueryIndexPlan("an_index", null, IndexScanComparisons.byValue(), @@ -78,7 +82,7 @@ void simpleIndexScanTest() throws Exception { ImmutableList.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.INT), Optional.of("field1")), Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("field2")))), QueryPlanConstraint.tautology(), - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); PlanSerializationContext planSerializationContext = PlanSerializationContext.newForCurrentMode(); final PPlanReference proto = planSerializationContext.toPlanReferenceProto(plan); final byte[] bytes = proto.toByteArray(); @@ -89,8 +93,9 @@ void simpleIndexScanTest() throws Exception { Assertions.assertTrue(plan.semanticEquals(parsedPlan)); } - @Test - void recordQueryDefaultOnEmptyPlanTest() throws Exception { + @ParameterizedTest + @EnumSource(KeyValueCursorBase.SerializationMode.class) + void recordQueryDefaultOnEmptyPlanTest(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { final RecordQueryIndexPlan indexPlan = new RecordQueryIndexPlan("an_index", null, IndexScanComparisons.byValue(), @@ -101,7 +106,7 @@ void recordQueryDefaultOnEmptyPlanTest() throws Exception { Optional.empty(), Type.primitiveType(Type.TypeCode.INT, true), QueryPlanConstraint.tautology(), - KeyValueCursorBase.SerializationMode.TO_OLD); + serializationMode); final var plan = new RecordQueryDefaultOnEmptyPlan(Quantifier .Physical.physicalBuilder().withAlias(CorrelationIdentifier.of("q42")).build( Reference.plannedOf(indexPlan) diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/Options.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/Options.java index e9ca6e747a..d066e39838 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/Options.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/Options.java @@ -194,7 +194,8 @@ public enum Name { * operations interacting with FDB. * Scope: Engine */ - ASYNC_OPERATIONS_TIMEOUT_MILLIS + ASYNC_OPERATIONS_TIMEOUT_MILLIS, + KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW, } public enum IndexFetchMethod { @@ -229,6 +230,7 @@ public enum IndexFetchMethod { builder.put(Name.CASE_SENSITIVE_IDENTIFIERS, false); builder.put(Name.CONTINUATIONS_CONTAIN_COMPILED_STATEMENTS, true); builder.put(Name.ASYNC_OPERATIONS_TIMEOUT_MILLIS, 10_000L); + builder.put(Name.KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW, false); OPTIONS_DEFAULT_VALUES = builder.build(); } @@ -364,6 +366,7 @@ private static Map> makeContracts() { data.put(Name.VALID_PLAN_HASH_MODES, List.of(TypeContract.stringType())); data.put(Name.CONTINUATIONS_CONTAIN_COMPILED_STATEMENTS, List.of(TypeContract.booleanType())); data.put(Name.ASYNC_OPERATIONS_TIMEOUT_MILLIS, List.of(TypeContract.longType(), RangeContract.of(0L, Long.MAX_VALUE))); + data.put(Name.KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW, List.of(TypeContract.booleanType())); return Collections.unmodifiableMap(data); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordLayerIterator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordLayerIterator.java index ed99e1130b..65deff5b79 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordLayerIterator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/RecordLayerIterator.java @@ -83,6 +83,7 @@ public boolean hasNext() { private void fetchNextResult() { if (result != null) { + continuation = ContinuationImpl.fromRecordCursorContinuation(result.getContinuation()); return; } result = recordCursor.getNext(); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java index 94a4bfc73c..7ade9d0c0e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java @@ -395,12 +395,14 @@ private static CascadesPlanner createPlanner(@Nonnull RecordMetaData metaData, @Nonnull Options options) throws RelationalException { // todo (yhatem) TODO (Interaction between planner configurations and query cache) Options.IndexFetchMethod indexFetchMethod = options.getOption(Options.Name.INDEX_FETCH_METHOD); + boolean kvCursorContinuationSerializeToNew = options.getOption(Options.Name.KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW); CascadesPlanner planner = new CascadesPlanner(metaData, recordStoreState); // TODO: TODO (Expose planner configuration parameters like index scan preference) RecordQueryPlannerConfiguration configuration = RecordQueryPlannerConfiguration.builder() .setIndexScanPreference(QueryPlanner.IndexScanPreference.PREFER_INDEX) .setIndexFetchMethod(toRecLayerIndexFetchMethod(indexFetchMethod)) .setAttemptFailedInJoinAsUnionMaxSize(24) + .setKeyValueCursorContinuationSerializationMode(kvCursorContinuationSerializeToNew) .build(); planner.setConfiguration(configuration); return planner; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java index ee44d48eb9..d02986cf06 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/QueryPlan.java @@ -375,6 +375,7 @@ private RelationalResultSet executePhysicalPlan(@Nonnull final RecordLayerSchema final var executeProperties = connection.getExecuteProperties().toBuilder() .setReturnedRowLimit(options.getOption(Options.Name.MAX_ROWS)) .setDryRun(options.getOption(Options.Name.DRY_RUN)) + .setKvCursorContSerializeToNew(options.getOption(Options.Name.KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW)) .build(); cursor = executionContext.metricCollector.clock( RelationalMetric.RelationalEvent.EXECUTE_RECORD_QUERY_PLAN, () -> recordQueryPlan.executePlan(fdbRecordStore, evaluationContext, diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java index e3e3cb85ff..c1bd38ffe3 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java @@ -119,48 +119,54 @@ void groupByForceContinuation() throws Exception { final String schemaTemplate = "create table t1(pk bigint, a bigint, b bigint, c bigint, primary key(pk))\n" + "create index mv1 as select count(*) from t1\n" + - "create index mv8 as select a, max(b) from t1 group by a order by a"; + "create index mv2 as select b from t1"; + Continuation continuation = null; try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { try (var conn = ddl.setSchemaAndGetConnection()) { conn.setOption(Options.Name.MAX_ROWS, 1); conn.setOption(Options.Name.CONTINUATIONS_CONTAIN_COMPILED_STATEMENTS, true); + conn.setOption(Options.Name.KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW, false); try (var statement = conn.createStatement()) { insertT1Record(statement, 1, 10, 1, 20); insertT1Record(statement, 2, 10, 2, 20); - insertT1Record(statement, 3, 10, 3, 5); - insertT1Record(statement, 4, 10, 4, 15); - insertT1Record(statement, 5, 10, 5, 5); - insertT1Record(statement, 6, 20, 6, 10); - insertT1Record(statement, 7, 20, 7, 40); - insertT1Record(statement, 8, 20, 8, 20); - insertT1Record(statement, 9, 20, 9, 90); - insertT1Record(statement, 10, 20, 10, 10); - insertT1Record(statement, 11, 20, 11, 40); - insertT1Record(statement, 12, 20, 12, 20); - insertT1Record(statement, 13, 20, 13, 90); + insertT1Record(statement, 3, 10, 2, 5); + insertT1Record(statement, 4, 10, 2, 15); + insertT1Record(statement, 5, 10, 2, 5); + insertT1Record(statement, 6, 20, 3, 10); + insertT1Record(statement, 7, 20, 4, 40); + insertT1Record(statement, 8, 20, 4, 20); + insertT1Record(statement, 9, 20, 4, 90); + insertT1Record(statement, 10, 20, 4, 10); + insertT1Record(statement, 11, 20, 4, 40); + insertT1Record(statement, 12, 20, 4, 20); + insertT1Record(statement, 13, 20, 4, 90); } - Continuation continuation = null; try (var statement = conn.createStatement()) { - conn.setOption(Options.Name.MAX_ROWS, 1); - conn.setOption(Options.Name.CONTINUATIONS_CONTAIN_COMPILED_STATEMENTS, true); - String query = "select count(*) from t1"; + String query = "select count(*) from t1 group by b"; Assertions.assertTrue(statement.execute(query), "Did not return a result set from a select statement!"); try (final RelationalResultSet resultSet = statement.getResultSet()) { - ResultSetAssert.assertThat(resultSet).hasNextRow() - .isRowExactly(13L) + ResultSetAssert.assertThat(resultSet) + .hasNextRow() + .isRowExactly(1L) .hasNoNextRow(); continuation = resultSet.getContinuation(); } } + } + } + try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { + try (var conn = ddl.setSchemaAndGetConnection()) { + conn.setOption(Options.Name.MAX_ROWS, 1); + conn.setOption(Options.Name.CONTINUATIONS_CONTAIN_COMPILED_STATEMENTS, true); + conn.setOption(Options.Name.KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW, false); try (var statement = conn.prepareStatement("EXECUTE CONTINUATION ?param")) { - conn.setOption(Options.Name.MAX_ROWS, 1); - conn.setOption(Options.Name.CONTINUATIONS_CONTAIN_COMPILED_STATEMENTS, true); statement.setMaxRows(1); statement.setBytes("param", continuation.serialize()); try (final RelationalResultSet resultSet = statement.executeQuery()) { - ResultSetAssert.assertThat(resultSet).hasNoNextRow(); + ResultSetAssert.assertThat(resultSet) + .hasNoNextRow(); continuation = resultSet.getContinuation(); - Assertions.assertTrue(continuation.getExecutionState().length == 0); + // Assertions.assertTrue(continuation.getExecutionState().length == 0); } } } diff --git a/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql b/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql index 3f31b84936..c04499e1ce 100644 --- a/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql +++ b/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql @@ -48,9 +48,8 @@ test_block: - query: select count(*) from t1 - explain: "AISCAN(MV1 <,> BY_GROUP -> [_0: VALUE:[0]]) | MAP (_ AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)" # Cannot run with FORCE_CONTINUATIONS due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 1 + - maxRows: 0 - result: [{4}] - - result: [] - - query: select count(*) from t1 group by col2 - explain: "AISCAN(MV2 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)" @@ -59,9 +58,8 @@ test_block: - query: select count(col1) from t1 - explain: "AISCAN(MV3 <,> BY_GROUP -> [_0: VALUE:[0]]) | MAP (_ AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)" # Cannot run with FORCE_CONTINUATIONS due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 1 + - maxRows: 0 - result: [{2}] - - result: [] - - query: select count(col1) from t1 group by col2 - explain: "AISCAN(MV4 <,> BY_GROUP -> [_0: KEY:[0], _1: VALUE:[0]]) | MAP (_._1 AS _0)" @@ -88,9 +86,8 @@ test_block: - query: select count(*) from t2 - explain: "ISCAN(MV5 <,>) | MAP (_ AS _0) | AGG (count_star(*) AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)" # Cannot run with FORCE_CONTINUATIONS due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 1 + - maxRows: 0 - result: [{4}] - - result: [] - - query: select count(*) from t2 group by col2 # Plan deserialization previously failed : https://github.com/FoundationDB/fdb-record-layer/issues/3214 @@ -112,9 +109,8 @@ test_block: - query: select count(col1) from t2 - explain: "ISCAN(MV5 <,>) | MAP (_ AS _0) | AGG (count(_._0.COL1) AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)" # Cannot run with FORCE_CONTINUATIONS due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 1 + - maxRows: 0 - result: [{2}] - - result: [] - - query: select count(col1) from t2 group by col2 # Plan deserialization previously failed : https://github.com/FoundationDB/fdb-record-layer/issues/3214 From 22a6e85bcb978c52d7c1a5152a11b4c938fb4b31 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Mon, 16 Jun 2025 15:56:06 -0700 Subject: [PATCH 11/15] save --- .../foundationdb/KeyValueCursorBase.java | 6 +- .../plan/RecordQueryPlannerConfiguration.java | 12 -- .../plan/plans/RecordQueryIndexPlan.java | 7 +- .../src/main/proto/record_cursor.proto | 5 +- .../RecordQueryIndexPlanWithOverScanTest.java | 2 - .../recordlayer/query/PlanGenerator.java | 1 - .../query/ForceContinuationQueryTests.java | 154 ++++++++++++++++++ .../aggregate-index-tests-count.yamsql | 4 - yaml-tests/src/test/resources/catalog.yamsql | 1 - 9 files changed, 164 insertions(+), 28 deletions(-) create mode 100644 fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ForceContinuationQueryTests.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java index e24e016148..d9869e8a5d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java @@ -169,6 +169,7 @@ public boolean isEnd() { @Nonnull @Override public ByteString toByteString() { + System.out.println("Keyvaluecursor toByteString serialization mode:" + serializationMode.name()); if (serializationMode == SerializationMode.TO_OLD) { if (lastKey == null) { return ByteString.EMPTY; @@ -228,11 +229,11 @@ public static byte[] fromRawBytes(@Nonnull byte[] rawBytes, SerializationMode se private RecordCursorProto.KeyValueCursorContinuation toProto() { RecordCursorProto.KeyValueCursorContinuation.Builder builder = RecordCursorProto.KeyValueCursorContinuation.newBuilder(); if (lastKey == null) { - builder.setIsEnd(true).setPrefixLength(prefixLength); + builder.setIsEnd(true); } else { ByteString base = ZeroCopyByteString.wrap(lastKey); builder.setContinuation(base.substring(prefixLength, lastKey.length)) - .setIsEnd(false).setPrefixLength(prefixLength); + .setIsEnd(false); } return builder.build(); } @@ -333,7 +334,6 @@ protected void prepare() { realContinuation = continuation; } } catch (InvalidProtocolBufferException ex) { - System.out.println("InvalidProtocolBufferException hit"); realContinuation = continuation; } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java index 17b79658b9..31a940a0e7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java @@ -440,18 +440,6 @@ public Builder() { this.protoBuilder = RecordPlannerConfigurationProto.PlannerConfiguration.newBuilder(); } - @Nonnull - public Builder setKeyValueCursorContinuationSerializationMode(boolean toNew) { - if (toNew) { - protoBuilder.setKeyValueCursorContinuationSerializationMode(RecordPlannerConfigurationProto.PlannerConfiguration.KeyValueCursorContinuationSerializationMode.TO_NEW); - updateFlags(true, KEYVALUE_CURSOR_CONTINUATION_SERIALIZATION_MODE); - } else { - protoBuilder.setKeyValueCursorContinuationSerializationMode(RecordPlannerConfigurationProto.PlannerConfiguration.KeyValueCursorContinuationSerializationMode.TO_OLD); - updateFlags(false, KEYVALUE_CURSOR_CONTINUATION_SERIALIZATION_MODE); - } - return this; - } - @Nonnull public Builder setIndexScanPreference(@Nonnull QueryPlanner.IndexScanPreference indexScanPreference) { protoBuilder.setIndexScanPreference(SCAN_PREFERENCE_BI_MAP.get(indexScanPreference)); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java index 89ef84e1e5..1e4e875b2d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java @@ -768,9 +768,12 @@ public static RecordQueryIndexPlan fromProto(@Nonnull final PlanSerializationCon private static class IndexScanContinuationConvertor implements RecordCursor.ContinuationConvertor { @Nonnull private final byte[] prefixBytes; + @Nonnull + private final KeyValueCursorBase.SerializationMode serializationMode; - public IndexScanContinuationConvertor(@Nonnull byte[] prefixBytes) { + public IndexScanContinuationConvertor(@Nonnull byte[] prefixBytes, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { this.prefixBytes = prefixBytes; + this.serializationMode = serializationMode; } @Nullable @@ -780,7 +783,7 @@ public byte[] unwrapContinuation(@Nullable final byte[] continuation) { return null; } // Add the prefix back to the inner continuation - byte[] innerContinuation = KeyValueCursorBase.Continuation.fromRawBytes(continuation, KeyValueCursorBase.SerializationMode.TO_NEW); + byte[] innerContinuation = KeyValueCursorBase.Continuation.fromRawBytes(continuation, serializationMode); return ByteArrayUtil.join(prefixBytes, innerContinuation); } diff --git a/fdb-record-layer-core/src/main/proto/record_cursor.proto b/fdb-record-layer-core/src/main/proto/record_cursor.proto index c806963b18..632390adcd 100644 --- a/fdb-record-layer-core/src/main/proto/record_cursor.proto +++ b/fdb-record-layer-core/src/main/proto/record_cursor.proto @@ -167,7 +167,6 @@ message RangeCursorContinuation { } message KeyValueCursorContinuation { - optional int32 prefixLength = 1; - optional bytes continuation = 2; - optional bool isEnd = 3; + optional bytes continuation = 1; + optional bool isEnd = 2; } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java index 5f56a10a5e..27dcd741a6 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlanWithOverScanTest.java @@ -67,7 +67,6 @@ import static org.hamcrest.Matchers.allOf; import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -81,7 +80,6 @@ class RecordQueryIndexPlanWithOverScanTest extends FDBRecordStoreQueryTestBase { private static final RecordQueryPlannerConfiguration plannerConfiguration = RecordQueryPlannerConfiguration.builder() .setIndexScanPreference(QueryPlanner.IndexScanPreference.PREFER_INDEX) .setAttemptFailedInJoinAsOr(true) - .setKeyValueCursorContinuationSerializationMode(true) .setComplexityThreshold(RecordQueryPlanner.DEFAULT_COMPLEXITY_THRESHOLD) .addValueIndexOverScanNeeded("MySimpleRecord$str_value_indexed") .addValueIndexOverScanNeeded("compoundIndex") diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java index 7ade9d0c0e..2ab8ea3814 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java @@ -402,7 +402,6 @@ private static CascadesPlanner createPlanner(@Nonnull RecordMetaData metaData, .setIndexScanPreference(QueryPlanner.IndexScanPreference.PREFER_INDEX) .setIndexFetchMethod(toRecLayerIndexFetchMethod(indexFetchMethod)) .setAttemptFailedInJoinAsUnionMaxSize(24) - .setKeyValueCursorContinuationSerializationMode(kvCursorContinuationSerializeToNew) .build(); planner.setConfiguration(configuration); return planner; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ForceContinuationQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ForceContinuationQueryTests.java new file mode 100644 index 0000000000..ba1e5b758b --- /dev/null +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ForceContinuationQueryTests.java @@ -0,0 +1,154 @@ +/* + * JoinWithLimitTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.query; + +import com.apple.foundationdb.relational.api.Continuation; +import com.apple.foundationdb.relational.api.Options; +import com.apple.foundationdb.relational.api.exceptions.ContextualSQLException; +import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalExtension; +import com.apple.foundationdb.relational.recordlayer.RelationalConnectionRule; +import com.apple.foundationdb.relational.recordlayer.RelationalStatementRule; +import com.apple.foundationdb.relational.recordlayer.UniqueIndexTests; +import com.apple.foundationdb.relational.recordlayer.Utils; +import com.apple.foundationdb.relational.utils.SimpleDatabaseRule; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.sql.SQLException; +import java.util.stream.Stream; + +public class ForceContinuationQueryTests { + + @RegisterExtension + @Order(0) + public final EmbeddedRelationalExtension relationalExtension = new EmbeddedRelationalExtension(); + + private static final String getTemplate_definition = + "create table t1(id bigint, col1 bigint, col2 bigint, primary key(id))\n" + + "create index mv1 as select count(*) from t1\n" + + "create index mv2 as select count(*) from t1 group by col2\n" + + "create index mv3 as select count(col1) from t1\n" + + "create index mv4 as select count(col1) from t1 group by col2\n" + + + "create table t2(id bigint, col1 bigint, col2 bigint, col3 bigint, primary key(id))\n" + + "create index mv7 as select min_ever(col3) from t2\n" + + + "create table t3(id bigint, col1 bigint, col2 bigint, primary key(id))\n" + + "create index t3_i1 as select count(*) from t3\n" + + "create index t3_i2 as select count(*) from t3 group by col1\n" + + "create index t3_i3 as select count(col2) from t3\n" + + "create index t3_i4 as select count(col2) from t3 group by col1\n" + + "create index t3_i5 as select sum(col1) from t3\n" + + "create index t3_i6 as select sum(col1) from t3 group by col2"; + + @RegisterExtension + @Order(1) + public final SimpleDatabaseRule db = new SimpleDatabaseRule(relationalExtension, UniqueIndexTests.class, getTemplate_definition); + + @RegisterExtension + @Order(2) + public final RelationalConnectionRule connection = new RelationalConnectionRule(db::getConnectionUri) + .withOptions(Options.builder().withOption(Options.Name.CONTINUATIONS_CONTAIN_COMPILED_STATEMENTS, true).build()) + .withSchema(db.getSchemaName()); + + @RegisterExtension + @Order(3) + public final RelationalStatementRule statement = new RelationalStatementRule(connection); + + public ForceContinuationQueryTests() throws SQLException { + } + + @BeforeAll + public static void beforeAll() { + Utils.enableCascadesDebugger(); + } + + @BeforeEach + void setup() throws Exception { + statement.execute("INSERT INTO T1 VALUES (1, 10, 1), (2, null, 2), (3, null, 2), (4, 12, 2)"); + statement.execute("INSERT INTO T2 VALUES (1, 10, 1, 11), (2, null, 2, 20), (3, null, 2, 20), (4, 12, 2, 20)"); + statement.execute("insert into t3 values (1, 2, 3), (2, 2, 3), (3, 2, 3)"); + statement.execute("delete from t3"); + } + + @ParameterizedTest + @MethodSource("provideArguments") + void testOldSerialization(String sql, long result) throws Exception { + Continuation continuation; + statement.setMaxRows(1); + try (var resultSet = statement.executeQuery(sql)) { + Assertions.assertTrue(resultSet.next()); + Assertions.assertEquals(result, resultSet.getLong(1)); + continuation = resultSet.getContinuation(); + } + // old kvCursorContinuation cause continuation at beginning exception + try (final var preparedStatement = connection.prepareStatement("EXECUTE CONTINUATION ?param")) { + preparedStatement.setMaxRows(1); + preparedStatement.setBytes("param", continuation.serialize()); + Assertions.assertThrows(ContextualSQLException.class, preparedStatement::executeQuery); + } + } + + @ParameterizedTest + @MethodSource("provideArguments") + void testNewSerialization(String sql, long result) throws Exception { + connection.setOption(Options.Name.KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW, true); + Continuation continuation; + try (final var s = connection.createStatement()) { + s.setMaxRows(1); + try (var resultSet = s.executeQuery(sql)) { + Assertions.assertTrue(resultSet.next()); + Assertions.assertEquals(result, resultSet.getLong(1)); + continuation = resultSet.getContinuation(); + } + } + // new serialization fixed the issue + try (final var preparedStatement = connection.prepareStatement("EXECUTE CONTINUATION ?param")) { + preparedStatement.setMaxRows(1); + preparedStatement.setBytes("param", continuation.serialize()); + try (var resultSet = preparedStatement.executeQuery()) { + Assertions.assertFalse(resultSet.next()); + } + } + } + + + private static Stream provideArguments() { + return Stream.of( + // aggregate-index-count.yamsql + Arguments.of("select count(*) from t1", 4L), + Arguments.of("select count(col1) from t1", 2L), + // aggregate-index-tests.yamsql + Arguments.of("select min_ever(col3) from t2", 11L), + // aggregate-empty-table.yamsql + Arguments.of("select count(*) from t3", 0L), + Arguments.of("select count(col2) from t3", 0L), + Arguments.of("select sum(col1) from t3", 0L) + ); + } +} diff --git a/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql b/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql index c04499e1ce..5ae240d176 100644 --- a/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql +++ b/yaml-tests/src/test/resources/aggregate-index-tests-count.yamsql @@ -85,8 +85,6 @@ test_block: - - query: select count(*) from t2 - explain: "ISCAN(MV5 <,>) | MAP (_ AS _0) | AGG (count_star(*) AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)" - # Cannot run with FORCE_CONTINUATIONS due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 0 - result: [{4}] - - query: select count(*) from t2 group by col2 @@ -108,8 +106,6 @@ test_block: - - query: select count(col1) from t2 - explain: "ISCAN(MV5 <,>) | MAP (_ AS _0) | AGG (count(_._0.COL1) AS _0) | ON EMPTY NULL | MAP (coalesce_long(_._0._0, promote(0l AS LONG)) AS _0)" - # Cannot run with FORCE_CONTINUATIONS due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - - maxRows: 0 - result: [{2}] - - query: select count(col1) from t2 group by col2 diff --git a/yaml-tests/src/test/resources/catalog.yamsql b/yaml-tests/src/test/resources/catalog.yamsql index 4b9ec7208b..e1aa8c7799 100644 --- a/yaml-tests/src/test/resources/catalog.yamsql +++ b/yaml-tests/src/test/resources/catalog.yamsql @@ -62,7 +62,6 @@ test_block: - query: select sum(cnt) from (select count(*) as cnt, template_name, template_version from schemas group by template_name, template_version having template_name = 'TEST_TEMPLATE_1') as t; - explain: "AISCAN(TEMPLATES_COUNT_INDEX [EQUALS promote(@c29 AS STRING)] BY_GROUP -> [_0: KEY:[0], _1: KEY:[1], _2: VALUE:[0]]) | MAP (_._2 AS CNT, _._0 AS TEMPLATE_NAME, _._1 AS TEMPLATE_VERSION) | MAP (_ AS _0) | AGG (sum_l(_._0.CNT) AS _0) | ON EMPTY NULL | MAP (_._0._0 AS _0)" - - maxRows: 0 # Disable force continuations because of empty continuation due to: https://github.com/FoundationDB/fdb-record-layer/issues/3206 - result: [{4}] - # How many schemas with the specified schemaTemplateName and schemaTemplateVersion exist in this cluster? From f30c3e3d4df2bb1614216b69de7e8f2bf2dbda16 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Mon, 16 Jun 2025 16:21:30 -0700 Subject: [PATCH 12/15] revert PlannerConfiguration change: --- .../record/query/plan/RecordQueryPlanner.java | 4 +- .../plan/RecordQueryPlannerConfiguration.java | 6 -- .../AggregateIndexMatchCandidate.java | 3 +- .../ValueIndexScanMatchCandidate.java | 6 +- .../WindowedIndexScanMatchCandidate.java | 6 +- .../plan/plans/RecordQueryIndexPlan.java | 35 +++------- .../foundationdb/QueryPlanCursorTest.java | 64 ++++++++---------- .../foundationdb/RecordTypeKeyTest.java | 2 +- .../MultidimensionalIndexTestBase.java | 65 ++++++++----------- .../SimpleMultidimensionalIndexTest.java | 39 ++++------- .../SlowMultidimensionalIndexTest.java | 43 ++++-------- .../limits/FDBRecordStoreLimitTestBase.java | 6 +- .../limits/FDBRecordStoreScanLimitTest.java | 7 +- .../query/FDBAndQueryToIntersectionTest.java | 13 ++-- .../query/FDBOrQueryToUnionTest.java | 13 ++-- ...ueryPlanStructuralInstrumentationTest.java | 36 +++++----- .../matchers/ExpressionMatcherTest.java | 13 ++-- .../plan/explain/ExplainPlanVisitorTest.java | 2 +- .../planning/RecordQueryPlanEqualityTest.java | 2 +- .../serialization/PlanSerializationTest.java | 16 ++--- 20 files changed, 145 insertions(+), 236 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlanner.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlanner.java index 83112e6fd6..f1bcd08a7f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlanner.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlanner.java @@ -1765,7 +1765,7 @@ private RecordQueryPlan planScan(@Nonnull CandidateScan candidateScan, fetchIndexRecords == FetchIndexRecords.PRIMARY_KEY ? getConfiguration().getIndexFetchMethod() : IndexFetchMethod.SCAN_AND_FETCH; - plan = new RecordQueryIndexPlan(candidateScan.index.getName(), candidateScan.planContext.commonPrimaryKey, indexScanParameters, indexFetchMethod, fetchIndexRecords, candidateScan.reverse, strictlySorted, configuration.getKeyValueCursorSerializationMode()); + plan = new RecordQueryIndexPlan(candidateScan.index.getName(), candidateScan.planContext.commonPrimaryKey, indexScanParameters, indexFetchMethod, fetchIndexRecords, candidateScan.reverse, strictlySorted); possibleTypes = getPossibleTypes(candidateScan.index); } // Add a type filter if the query plan might return records of more types than the query specified @@ -2158,7 +2158,7 @@ public RecordQueryCoveringIndexPlan planCoveringAggregateIndex(@Nonnull RecordQu RecordQueryIndexPlan plan = (RecordQueryIndexPlan)scoredPlan.getPlan(); IndexScanParameters scanParameters = new IndexScanComparisons(IndexScanType.BY_GROUP, plan.getScanComparisons()); - plan = new RecordQueryIndexPlan(plan.getIndexName(), scanParameters, plan.isReverse(), configuration.getKeyValueCursorSerializationMode()); + plan = new RecordQueryIndexPlan(plan.getIndexName(), scanParameters, plan.isReverse()); return new RecordQueryCoveringIndexPlan(plan, recordType.getName(), AvailableFields.NO_FIELDS, builder.build()); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java index 31a940a0e7..0a64b89981 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/RecordQueryPlannerConfiguration.java @@ -23,7 +23,6 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.IndexFetchMethod; import com.apple.foundationdb.record.RecordPlannerConfigurationProto; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.plan.cascades.CascadesRule; import com.apple.foundationdb.record.query.plan.cascades.PlannerRule; @@ -66,7 +65,6 @@ public class RecordQueryPlannerConfiguration { private static final long PLAN_OTHER_ATTEMPT_FULL_FILTER_MASK = 1L << 9; private static final long NORMALIZE_NESTED_FIELDS_MASK = 1L << 10; private static final long OMIT_PRIMARY_KEY_IN_ORDERING_KEY_FOR_IN_UNION_MASK = 1L << 11; - private static final long KEYVALUE_CURSOR_CONTINUATION_SERIALIZATION_MODE = 1L << 12; @Nonnull private final RecordPlannerConfigurationProto.PlannerConfiguration proto; @@ -230,10 +228,6 @@ public boolean shouldOptimizeForRequiredResults() { return flagSet(OPTIMIZE_FOR_REQUIRED_RESULTS_MASK); } - public KeyValueCursorBase.SerializationMode getKeyValueCursorSerializationMode() { - return flagSet(KEYVALUE_CURSOR_CONTINUATION_SERIALIZATION_MODE) ? KeyValueCursorBase.SerializationMode.TO_NEW : KeyValueCursorBase.SerializationMode.TO_OLD; - } - /** * Return the size limit of the cascades planner task queue. * @return the maximum size of the queue. 0 means "unbound" (the default). Trying to add a task beyond the maximum size will fail the planning. diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexMatchCandidate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexMatchCandidate.java index ac26ab3efb..11b5dcc4e6 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexMatchCandidate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexMatchCandidate.java @@ -383,8 +383,7 @@ public RecordQueryPlan toEquivalentPlan(@Nonnull final PartialMatch partialMatch false, partialMatch.getMatchCandidate(), baseRecordType, - QueryPlanConstraint.tautology(), - planContext.getPlannerConfiguration().getKeyValueCursorSerializationMode()); + QueryPlanConstraint.tautology()); return new RecordQueryAggregateIndexPlan(aggregateIndexScan, recordTypes.get(0).getName(), diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java index 6d6da35a1c..d861839d57 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexScanMatchCandidate.java @@ -245,8 +245,7 @@ public RecordQueryPlan toEquivalentPlan(@Nonnull final PartialMatch partialMatch false, partialMatch.getMatchCandidate(), baseRecordType, - matchInfo.getConstraint(), - planContext.getPlannerConfiguration().getKeyValueCursorSerializationMode())); + matchInfo.getConstraint())); } @Nonnull @@ -273,8 +272,7 @@ private Optional tryFetchCoveringIndexScan(@Nonnull final Parti false, partialMatch.getMatchCandidate(), baseRecordType, - partialMatch.getRegularMatchInfo().getConstraint(), - planContext.getPlannerConfiguration().getKeyValueCursorSerializationMode()); + partialMatch.getRegularMatchInfo().getConstraint()); final var coveringIndexPlan = new RecordQueryCoveringIndexPlan(indexPlan, indexEntryToLogicalRecord.getQueriedRecordType().getName(), diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java index 9d23b6e611..abab9990bd 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/WindowedIndexScanMatchCandidate.java @@ -403,8 +403,7 @@ public RecordQueryPlan toEquivalentPlan(@Nonnull final PartialMatch partialMatch false, partialMatch.getMatchCandidate(), baseRecordType, - QueryPlanConstraint.tautology(), - planContext.getPlannerConfiguration().getKeyValueCursorSerializationMode())); + QueryPlanConstraint.tautology())); } @Nonnull @@ -430,8 +429,7 @@ private Optional tryFetchCoveringIndexScan(@Nonnull final Parti false, partialMatch.getMatchCandidate(), baseRecordType, - QueryPlanConstraint.tautology(), - planContext.getPlannerConfiguration().getKeyValueCursorSerializationMode()); + QueryPlanConstraint.tautology()); final var coveringIndexPlan = new RecordQueryCoveringIndexPlan(indexPlan, indexEntryToLogicalRecord.getQueriedRecordType().getName(), diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java index 1e4e875b2d..7e1811009b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java @@ -165,11 +165,9 @@ public class RecordQueryIndexPlan implements RecordQueryPlanWithNoChildren, @Nonnull private final Supplier comparisonRangesSupplier; - @Nonnull - private final KeyValueCursorBase.SerializationMode serializationMode; - public RecordQueryIndexPlan(@Nonnull final String indexName, @Nonnull final IndexScanParameters scanParameters, final boolean reverse, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { - this(indexName, null, scanParameters, IndexFetchMethod.SCAN_AND_FETCH, FetchIndexRecords.PRIMARY_KEY, reverse, false, serializationMode); + public RecordQueryIndexPlan(@Nonnull final String indexName, @Nonnull final IndexScanParameters scanParameters, final boolean reverse) { + this(indexName, null, scanParameters, IndexFetchMethod.SCAN_AND_FETCH, FetchIndexRecords.PRIMARY_KEY, reverse, false); } public RecordQueryIndexPlan(@Nonnull final String indexName, @@ -178,9 +176,8 @@ public RecordQueryIndexPlan(@Nonnull final String indexName, @Nonnull final IndexFetchMethod useIndexPrefetch, @Nonnull final FetchIndexRecords fetchIndexRecords, final boolean reverse, - final boolean strictlySorted, - @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { - this(indexName, commonPrimaryKey, scanParameters, useIndexPrefetch, fetchIndexRecords, reverse, strictlySorted, Optional.empty(), new Type.Any(), QueryPlanConstraint.tautology(), serializationMode); + final boolean strictlySorted) { + this(indexName, commonPrimaryKey, scanParameters, useIndexPrefetch, fetchIndexRecords, reverse, strictlySorted, Optional.empty(), new Type.Any(), QueryPlanConstraint.tautology()); } public RecordQueryIndexPlan(@Nonnull final String indexName, @@ -192,9 +189,8 @@ public RecordQueryIndexPlan(@Nonnull final String indexName, final boolean strictlySorted, @Nonnull final MatchCandidate matchCandidate, @Nonnull final Type.Record resultType, - @Nonnull final QueryPlanConstraint constraint, - @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { - this(indexName, commonPrimaryKey, scanParameters, indexFetchMethod, fetchIndexRecords, reverse, strictlySorted, Optional.of(matchCandidate), resultType, constraint, serializationMode); + @Nonnull final QueryPlanConstraint constraint) { + this(indexName, commonPrimaryKey, scanParameters, indexFetchMethod, fetchIndexRecords, reverse, strictlySorted, Optional.of(matchCandidate), resultType, constraint); } protected RecordQueryIndexPlan(@Nonnull final PlanSerializationContext serializationContext, @@ -211,19 +207,6 @@ protected RecordQueryIndexPlan(@Nonnull final PlanSerializationContext serializa QueryPlanConstraint.fromProto(serializationContext, Objects.requireNonNull(recordQueryIndexPlanProto.getConstraint()))); } - protected RecordQueryIndexPlan(@Nonnull final String indexName, - @Nullable final KeyExpression commonPrimaryKey, - @Nonnull final IndexScanParameters scanParameters, - @Nonnull final IndexFetchMethod indexFetchMethod, - @Nonnull final FetchIndexRecords fetchIndexRecords, - final boolean reverse, - final boolean strictlySorted, - @Nonnull final Optional matchCandidateOptional, - @Nonnull final Type resultType, - @Nonnull final QueryPlanConstraint constraint) { - this(indexName, commonPrimaryKey, scanParameters, indexFetchMethod, fetchIndexRecords, reverse, strictlySorted, matchCandidateOptional, resultType, constraint, KeyValueCursorBase.SerializationMode.TO_OLD); - } - @VisibleForTesting public RecordQueryIndexPlan(@Nonnull final String indexName, @Nullable final KeyExpression commonPrimaryKey, @@ -234,8 +217,7 @@ public RecordQueryIndexPlan(@Nonnull final String indexName, final boolean strictlySorted, @Nonnull final Optional matchCandidateOptional, @Nonnull final Type resultType, - @Nonnull final QueryPlanConstraint constraint, - @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { + @Nonnull final QueryPlanConstraint constraint) { this.indexName = indexName; this.commonPrimaryKey = commonPrimaryKey; this.scanParameters = scanParameters; @@ -253,7 +235,6 @@ public RecordQueryIndexPlan(@Nonnull final String indexName, } this.constraint = constraint; this.comparisonRangesSupplier = Suppliers.memoize(this::computeComparisonRanges); - this.serializationMode = serializationMode; } @Nonnull @@ -351,7 +332,7 @@ private RecordCursor executeEntriesWithOverScan( @Nonnull FDBRecordStoreBase store, @Nonnull Index index, @Nullable byte[] continuation, @Nonnull ExecuteProperties executeProperties) { final byte[] prefixBytes = getRangePrefixBytes(tupleScanRange); - final IndexScanContinuationConvertor continuationConvertor = new IndexScanContinuationConvertor(prefixBytes); + final IndexScanContinuationConvertor continuationConvertor = new IndexScanContinuationConvertor(prefixBytes, executeProperties.isKvCursorContSerializeToNew() ? KeyValueCursorBase.SerializationMode.TO_NEW : KeyValueCursorBase.SerializationMode.TO_OLD); // Scan a wider range, and then halt when either this scans outside the given range final IndexScanRange newScanRange = new IndexScanRange(IndexScanType.BY_VALUE, widenedScanRange); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/QueryPlanCursorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/QueryPlanCursorTest.java index 0a24a4d3be..ba68e638d6 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/QueryPlanCursorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/QueryPlanCursorTest.java @@ -46,8 +46,6 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -153,10 +151,10 @@ private RecordQueryPlan scanPlan() { return new RecordQueryScanPlan(ScanComparisons.EMPTY, false); } - private RecordQueryPlan indexPlanEquals(String indexName, Object value, KeyValueCursorBase.SerializationMode serializationMode) { + private RecordQueryPlan indexPlanEquals(String indexName, Object value) { IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, value)), Collections.emptySet())); - return new RecordQueryIndexPlan(indexName, scan, false, serializationMode); + return new RecordQueryIndexPlan(indexName, scan, false); } private KeyExpression primaryKey() { @@ -181,10 +179,9 @@ public void indexlessFilter() throws Exception { compareSkipsAndCursors(plan); } - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - public void indexed(KeyValueCursorBase.SerializationMode serializationMode) throws Exception { - final RecordQueryPlan plan = indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2, serializationMode); + @Test + public void indexed() throws Exception { + final RecordQueryPlan plan = indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2); compareSkipsAndCursors(plan); } @@ -193,7 +190,7 @@ public void indexRange() throws Exception { final IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Collections.emptyList(), ImmutableSet.of( new Comparisons.SimpleComparison(Comparisons.Type.GREATER_THAN, 2), new Comparisons.SimpleComparison(Comparisons.Type.LESS_THAN, 4)))); - final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", scan, false, KeyValueCursorBase.SerializationMode.TO_OLD); + final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", scan, false); compareSkipsAndCursors(plan); } @@ -202,7 +199,7 @@ public void reverse() throws Exception { final IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Collections.emptyList(), ImmutableSet.of( new Comparisons.SimpleComparison(Comparisons.Type.GREATER_THAN, 2), new Comparisons.SimpleComparison(Comparisons.Type.LESS_THAN, 4)))); - final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", scan, true, KeyValueCursorBase.SerializationMode.TO_OLD); + final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", scan, true); compareSkipsAndCursors(plan); } @@ -210,62 +207,58 @@ public void reverse() throws Exception { public void in() throws Exception { final IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.ParameterComparison(Comparisons.Type.EQUALS, "in_num")), Collections.emptySet())); final RecordQueryPlan plan = new RecordQueryInValuesJoinPlan( - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", scan, false, KeyValueCursorBase.SerializationMode.TO_OLD), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", scan, false), "in_num", Bindings.Internal.IN, Arrays.asList(2, 4), false, false); compareSkipsAndCursors(plan); } - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - public void union(KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + @Test + public void union() throws Exception { final RecordQueryPlan plan = RecordQueryUnionPlan.from( - indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2, serializationMode), - indexPlanEquals("MySimpleRecord$num_value_3_indexed", 4, serializationMode), + indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2), + indexPlanEquals("MySimpleRecord$num_value_3_indexed", 4), primaryKey(), false); compareSkipsAndCursors(plan); } - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - public void unionOneSideAtATime(KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + @Test + public void unionOneSideAtATime() throws Exception { final RecordQueryPlan plan = RecordQueryUnionPlan.from( - indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2, serializationMode), - indexPlanEquals("MySimpleRecord$num_value_3_indexed", 4, serializationMode), + indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2), + indexPlanEquals("MySimpleRecord$num_value_3_indexed", 4), Key.Expressions.concat(Key.Expressions.field("num_value_3_indexed"), primaryKey()), true); compareSkipsAndCursors(plan); } - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - public void intersection(KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + @Test + public void intersection() throws Exception { final RecordQueryPlan plan = RecordQueryIntersectionPlan.from( - indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2, serializationMode), - indexPlanEquals("MySimpleRecord$str_value_indexed", "even", serializationMode), + indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2), + indexPlanEquals("MySimpleRecord$str_value_indexed", "even"), primaryKey()); compareSkipsAndCursors(plan); } - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - public void filter(KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + @Test + public void filter() throws Exception { final RecordQueryPlan plan = new RecordQueryFilterPlan( - indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2, serializationMode), + indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2), Query.field("str_value_indexed").equalsValue("even") ); compareSkipsAndCursors(plan); } - private void filterKeyCount(int amount, KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + private void filterKeyCount(int amount) throws Exception { final QueryComponent filter = Query.field("str_value_indexed").equalsValue("even"); try (FDBRecordContext context = openContext()) { openSimpleRecordStore(context); recordStore.getTimer().reset(); - final RecordQueryPlan plan = indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2, serializationMode); + final RecordQueryPlan plan = indexPlanEquals("MySimpleRecord$num_value_3_indexed", 2); byte[] continuation = null; int unfilteredCount = 0; @@ -307,11 +300,10 @@ private void filterKeyCount(int amount, KeyValueCursorBase.SerializationMode ser } } - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - public void filterKeyCount(KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + @Test + public void filterKeyCount() throws Exception { for (int amount : amounts) { - filterKeyCount(amount, serializationMode); + filterKeyCount(amount); } } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java index 9cec274cf0..46aec50a58 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/RecordTypeKeyTest.java @@ -257,7 +257,7 @@ public void testIndexScanOnSecondColumn() throws Exception { new RecordTypeKeyComparison("MySimpleRecord").getComparison() ), Collections.emptySet()); IndexScanParameters scan = IndexScanComparisons.byValue(comparison); - RecordQueryPlan plan = new RecordQueryIndexPlan(index.getName(), scan, false, KeyValueCursorBase.SerializationMode.TO_OLD); + RecordQueryPlan plan = new RecordQueryIndexPlan(index.getName(), scan, false); assertEquals(recs.subList(1, 2), recordStore.executeQuery(query) .map(FDBQueriedRecord::getStoredRecord).asList().join()); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java index 99edc250d4..13e905f357 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java @@ -497,7 +497,7 @@ void basicReadWithNullsTest(final boolean useAsync, @Nonnull final String storag } void indexReadTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -511,8 +511,7 @@ void indexReadTest(final boolean useAsync, final long seed, final int numRecords new HypercubeScanParameters("business", (Long)null, intervalEndInclusive, intervalStartInclusive, null), - false, - serializationMode); + false); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -550,7 +549,7 @@ void indexReadTest(final boolean useAsync, final long seed, final int numRecords } void indexReadWithNullsTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -564,8 +563,7 @@ void indexReadWithNullsTest(final boolean useAsync, final long seed, final int n new HypercubeScanParameters("business", (Long)null, intervalEndInclusive, intervalStartInclusive, null), - false, - serializationMode); + false); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -608,7 +606,7 @@ void indexReadWithNullsTest(final boolean useAsync, final long seed, final int n Assertions.assertEquals(expectedResults, actualResults); } - void indexReadWithNullsAndMinsTest1(final boolean useAsync, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + void indexReadWithNullsAndMinsTest1(final boolean useAsync) throws Exception { RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -620,8 +618,7 @@ void indexReadWithNullsAndMinsTest1(final boolean useAsync, @Nonnull final KeyVa new HypercubeScanParameters("business", (Long)null, 0L, 0L, null), - false, - serializationMode); + false); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -664,7 +661,7 @@ void indexReadWithNullsAndMinsTest1(final boolean useAsync, @Nonnull final KeyVa Assertions.assertEquals(expectedResults, actualResults); } - void indexReadWithNullsAndMinsTest2(final boolean useAsync, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + void indexReadWithNullsAndMinsTest2(final boolean useAsync) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -676,8 +673,7 @@ void indexReadWithNullsAndMinsTest2(final boolean useAsync, @Nonnull final KeyVa new HypercubeScanParameters("business", Long.MIN_VALUE, 0L, 0L, null), - false, - serializationMode); + false); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -715,7 +711,7 @@ void indexReadWithNullsAndMinsTest2(final boolean useAsync, @Nonnull final KeyVa } void indexReadIsNullTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -727,8 +723,7 @@ void indexReadIsNullTest(final boolean useAsync, final long seed, final int numR new HypercubeScanParameters("business", (Long)null, Long.MIN_VALUE, null, null), - false, - serializationMode); + false); final Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -755,7 +750,7 @@ void indexReadIsNullTest(final boolean useAsync, final long seed, final int numR Assertions.assertTrue(plan2.hasIndexScan("calendarNameStartEpoch")); } - void indexReadWithIn(final boolean useAsync, final long seed, final int numRecords, final int numIns, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + void indexReadWithIn(final boolean useAsync, final long seed, final int numRecords, final int numIns) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -794,8 +789,7 @@ void indexReadWithIn(final boolean useAsync, final long seed, final int numRecor new RecordQueryIndexPlan("EventIntervals", new CompositeScanParameters( new MultidimensionalIndexScanBounds(TupleRange.allOf(Tuple.from("business")), andBounds, TupleRange.ALL)), - false, - serializationMode); + false); final Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -827,7 +821,7 @@ void indexReadWithIn(final boolean useAsync, final long seed, final int numRecor void indexReadsAfterDeletesTest(final boolean useAsync, final long seed, final int numRecords, final int numDeletes, @Nonnull final String storage, final boolean storeHilbertValues, - final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + final boolean useNodeSlotIndex) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -842,8 +836,7 @@ void indexReadsAfterDeletesTest(final boolean useAsync, final long seed, final i new HypercubeScanParameters("business", (Long)null, intervalEndInclusive, intervalStartInclusive, null), - false, - serializationMode); + false); final Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -864,7 +857,7 @@ void indexReadsAfterDeletesTest(final boolean useAsync, final long seed, final i } void indexSkipScanTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -879,8 +872,7 @@ void indexSkipScanTest(final boolean useAsync, final long seed, final int numRec new HypercubeScanParameters("business", "private", null, intervalEndInclusive, intervalStartInclusive, null), - false, - serializationMode); + false); final Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = @@ -910,7 +902,7 @@ void indexSkipScanTest(final boolean useAsync, final long seed, final int numRec Assertions.assertTrue(plan2.hasRecordScan()); } - void continuationTest(final boolean useAsync, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + void continuationTest(final boolean useAsync) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -925,8 +917,7 @@ void continuationTest(final boolean useAsync, @Nonnull final KeyValueCursorBase. new HypercubeScanParameters("business", "private", Long.MIN_VALUE, intervalEndInclusive, intervalStartInclusive, null), - false, - serializationMode); + false); Set actualResults = getResultsWithContinuations(additionalIndexes, indexPlan, 4); @@ -984,7 +975,7 @@ private Set getResultsWithContinuations(@Nonnull final RecordMetaDataHo } void coveringIndexScanWithFetchTest(final boolean useAsync, @Nonnull final String storage, final boolean storeHilbertValues, - final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + final boolean useNodeSlotIndex) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -999,8 +990,7 @@ void coveringIndexScanWithFetchTest(final boolean useAsync, @Nonnull final Strin new HypercubeScanParameters("business", "private", null, intervalEndInclusive, intervalStartInclusive, null), - false, - serializationMode); + false); final RecordType myMultidimensionalRecord = recordStore.getRecordMetaData().getRecordType("MyMultidimensionalRecord"); final Index index = recordStore.getRecordMetaData().getIndex("EventIntervals"); @@ -1104,7 +1094,7 @@ void coveringIndexReadTest(final boolean useAsync, final long seed, final int nu } void indexScan3DTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { final RecordMetaDataHook additionalIndex = metaDataBuilder -> metaDataBuilder.addIndex("MyMultidimensionalRecord", new Index("EventIntervals3D", DimensionsKeyExpression.of(field("calendar_name"), @@ -1123,8 +1113,7 @@ void indexScan3DTest(final boolean useAsync, final long seed, final int numRecor Long.MIN_VALUE, intervalEndInclusive, intervalStartInclusive, null, epochMean + expirationCutOff, null), - false, - serializationMode); + false); Set actualResults = getResults(additionalIndex, indexPlan); final QueryComponent filter = @@ -1180,7 +1169,7 @@ void wrongDimensionTypes(final boolean useAsync) { Assertions.assertEquals(KeyExpression.InvalidExpressionException.class, cause.getClass()); } - void deleteWhereTest(final boolean useAsync, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + void deleteWhereTest(final boolean useAsync, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { metaDataBuilder.addIndex("MyMultidimensionalRecord", @@ -1211,15 +1200,14 @@ void deleteWhereTest(final boolean useAsync, @Nonnull final String storage, fina new RecordQueryIndexPlan("EventIntervals", new CompositeScanParameters( new MultidimensionalIndexScanBounds(TupleRange.allOf(Tuple.from(null, "business")), bounds, TupleRange.ALL)), - false, - serializationMode); + false); final Set actualResults = getResults(additionalIndexes, indexPlan); Assertions.assertTrue(actualResults.isEmpty()); } void unprefixedIndexReadTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex, @Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); @@ -1233,8 +1221,7 @@ void unprefixedIndexReadTest(final boolean useAsync, final long seed, final int new HypercubeScanParameters(null, (Long)null, intervalEndInclusive, intervalStartInclusive, null), - false, - serializationMode); + false); Set actualResults = getResults(additionalIndexes, indexPlan); final QueryComponent filter = diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java index 23c428edbd..cfe1bfa7be 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java @@ -179,40 +179,35 @@ void basicReadWithNullsTest(@Nonnull final String storage, final boolean storeHi @MethodSource("argumentsForBasicReads") void deleteWhereTest(@Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.deleteWhereTest(false, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.deleteWhereTest(false, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.deleteWhereTest(false, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @MethodSource("argumentsForBasicReads") void coveringIndexScanWithFetchTest(@Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.coveringIndexScanWithFetchTest(false, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.coveringIndexScanWithFetchTest(false, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.coveringIndexScanWithFetchTest(false, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void indexReadTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void indexReadWithNullsTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadWithNullsTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadWithNullsTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadWithNullsTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void indexReadIsNullTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadIsNullTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadIsNullTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadIsNullTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @@ -226,16 +221,14 @@ void coveringIndexReadTest(final long seed, final int numRecords, @Nonnull final @MethodSource("argumentsForIndexReads") void indexScan3DTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexScan3DTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexScan3DTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexScan3DTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void unprefixedIndexReadTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.unprefixedIndexReadTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.unprefixedIndexReadTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.unprefixedIndexReadTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @@ -263,8 +256,7 @@ void unprefixedSuffixedIndexReadWithResidualsTest(final long seed, final int num @MethodSource("argumentsForIndexReads") void indexSkipScanTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexSkipScanTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexSkipScanTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexSkipScanTest(false, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @@ -272,8 +264,7 @@ void indexSkipScanTest(final long seed, final int numRecords, @Nonnull final Str void indexReadsAfterDeletesTest(final long seed, final int numRecords, final int numDeletes, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadsAfterDeletesTest(false, seed, numRecords, numDeletes, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadsAfterDeletesTest(false, seed, numRecords, numDeletes, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadsAfterDeletesTest(false, seed, numRecords, numDeletes, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @@ -285,8 +276,7 @@ void indexReadWithDuplicatesTest(final int numRecords, @Nonnull final String sto @Test void continuationTest() throws Exception { - super.continuationTest(false, KeyValueCursorBase.SerializationMode.TO_OLD); - super.continuationTest(false, KeyValueCursorBase.SerializationMode.TO_NEW); + super.continuationTest(false); } @Test @@ -296,20 +286,17 @@ void wrongDimensionTypes() { @Test void indexReadWithNullsAndMinsTest1() throws Exception { - super.indexReadWithNullsAndMinsTest1(false, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadWithNullsAndMinsTest1(false, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadWithNullsAndMinsTest1(false); } @Test void indexReadWithNullsAndMinsTest2() throws Exception { - super.indexReadWithNullsAndMinsTest2(false, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadWithNullsAndMinsTest2(false, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadWithNullsAndMinsTest2(false); } @ParameterizedTest @MethodSource("argumentsForIndexReadWithIn") void indexReadWithIn(final long seed, final int numRecords, final int numIns) throws Exception { - super.indexReadWithIn(false, seed, numRecords, numIns, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadWithIn(false, seed, numRecords, numIns, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadWithIn(false, seed, numRecords, numIns); } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java index 10a021bce6..366e84440c 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java @@ -172,40 +172,35 @@ void basicReadWithNulls(@Nonnull final String storage, final boolean storeHilber @ParameterizedTest @MethodSource("argumentsForBasicReads") void deleteWhereTest(@Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.deleteWhereTest(true, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.deleteWhereTest(true, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.deleteWhereTest(true, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @MethodSource("argumentsForBasicReads") void coveringIndexScanWithFetchTest(@Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.coveringIndexScanWithFetchTest(true, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.coveringIndexScanWithFetchTest(true, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.coveringIndexScanWithFetchTest(true, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void indexReadTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void indexReadWithNullsTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadWithNullsTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadWithNullsTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadWithNullsTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void indexReadIsNullTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadIsNullTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadIsNullTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadIsNullTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @@ -219,16 +214,14 @@ void coveringIndexReadTest(final long seed, final int numRecords, @Nonnull final @MethodSource("argumentsForIndexReads") void indexScan3DTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexScan3DTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexScan3DTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexScan3DTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @MethodSource("argumentsForIndexReads") void unprefixedIndexReadTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.unprefixedIndexReadTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.unprefixedIndexReadTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.unprefixedIndexReadTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @@ -256,8 +249,7 @@ void unprefixedSuffixedIndexReadWithResidualsTest(final long seed, final int num @MethodSource("argumentsForIndexReads") void indexSkipScanTest(final long seed, final int numRecords, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexSkipScanTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexSkipScanTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexSkipScanTest(true, seed, numRecords, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @@ -265,9 +257,7 @@ void indexSkipScanTest(final long seed, final int numRecords, @Nonnull final Str void indexReadsAfterDeletesTest(final long seed, final int numRecords, final int numDeletes, @Nonnull final String storage, final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { - super.indexReadsAfterDeletesTest(true, seed, numRecords, numDeletes, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadsAfterDeletesTest(true, seed, numRecords, numDeletes, storage, storeHilbertValues, useNodeSlotIndex, KeyValueCursorBase.SerializationMode.TO_NEW); - + super.indexReadsAfterDeletesTest(true, seed, numRecords, numDeletes, storage, storeHilbertValues, useNodeSlotIndex); } @ParameterizedTest @@ -279,8 +269,7 @@ void indexReadWithDuplicatesTest(final int numRecords, @Nonnull final String sto @Test void continuationTest() throws Exception { - super.continuationTest(true, KeyValueCursorBase.SerializationMode.TO_OLD); - super.continuationTest(true, KeyValueCursorBase.SerializationMode.TO_NEW); + super.continuationTest(true); } @Test @@ -290,21 +279,18 @@ void wrongDimensionTypes() { @Test void indexReadWithNullsAndMinsTest1() throws Exception { - super.indexReadWithNullsAndMinsTest1(true, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadWithNullsAndMinsTest1(true, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadWithNullsAndMinsTest1(true); } @Test void indexReadWithNullsAndMinsTest2() throws Exception { - super.indexReadWithNullsAndMinsTest2(true, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadWithNullsAndMinsTest2(true, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadWithNullsAndMinsTest2(true); } @ParameterizedTest @MethodSource("argumentsForIndexReadWithIn") void indexReadWithIn(final long seed, final int numRecords, final int numIns) throws Exception { - super.indexReadWithIn(false, seed, numRecords, numIns, KeyValueCursorBase.SerializationMode.TO_OLD); - super.indexReadWithIn(false, seed, numRecords, numIns, KeyValueCursorBase.SerializationMode.TO_NEW); + super.indexReadWithIn(false, seed, numRecords, numIns); } @ParameterizedTest @@ -317,8 +303,7 @@ void concurrentReadsAndWrites(@Nonnull final String storage, final boolean store new HypercubeScanParameters("business", (Long)null, null, null, null), - false, - KeyValueCursorBase.SerializationMode.TO_OLD); + false); final var random = new Random(System.currentTimeMillis()); final var expectedMessages = new HashSet(); var writeNum = 0; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreLimitTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreLimitTestBase.java index c220bb632c..532cd50c80 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreLimitTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreLimitTestBase.java @@ -76,7 +76,7 @@ void setupSimpleRecordStore() throws Exception { private static RecordQueryPlan indexPlanEquals(String indexName, Object value) { IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, value)), Collections.emptySet())); - return new RecordQueryIndexPlan(indexName, scan, false, KeyValueCursorBase.SerializationMode.TO_OLD); + return new RecordQueryIndexPlan(indexName, scan, false); } private static KeyExpression primaryKey() { @@ -87,7 +87,7 @@ static Stream plans(boolean fail) { RecordQueryPlan scanPlan = new RecordQueryScanPlan(ScanComparisons.EMPTY, false); IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); RecordQueryPlan indexPlan = new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", - fullValueScan, false, KeyValueCursorBase.SerializationMode.TO_OLD); + fullValueScan, false); QueryComponent filter = Query.field("str_value_indexed").equalsValue("odd"); QueryComponent middleFilter = Query.and( Query.field("rec_no").greaterThan(24L), @@ -97,7 +97,7 @@ static Stream plans(boolean fail) { return Stream.of( Arguments.of("full record scan", fail, scanPlan), Arguments.of("simple index scan", fail, indexPlan), - Arguments.of("reverse index scan", fail, new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, true, KeyValueCursorBase.SerializationMode.TO_OLD)), + Arguments.of("reverse index scan", fail, new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, true)), Arguments.of("filter on scan plan", fail, new RecordQueryFilterPlan(scanPlan, filter)), Arguments.of("filter on index plan", fail, new RecordQueryFilterPlan(indexPlan, filter)), Arguments.of("type filter on scan plan", fail, new RecordQueryTypeFilterPlan(scanPlan, Collections.singletonList("MySimpleRecord"))), diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java index ab40c12b35..ab58e00731 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java @@ -351,14 +351,13 @@ void testSplitContinuation() { @ParameterizedTest @ValueSource(ints = {2, 5, 10, 20}) // for this test, the scan limit must divide 100 public void testExecuteStateReset(int scanLimit) throws Exception { - executeStateReset(scanLimit, KeyValueCursorBase.SerializationMode.TO_OLD); - executeStateReset(scanLimit, KeyValueCursorBase.SerializationMode.TO_NEW); + executeStateReset(scanLimit); } - private void executeStateReset(int scanLimit, @Nonnull KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + private void executeStateReset(int scanLimit) throws Exception { final IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", - fullValueScan, false, serializationMode); + fullValueScan, false); ExecuteProperties properties = ExecuteProperties.newBuilder().setScannedRecordsLimit(scanLimit).build(); try (FDBRecordContext context = openContext()) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java index e6f7bf4a4a..f3147fbdc3 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java @@ -966,16 +966,15 @@ public void sortedIntersectionBounded(boolean shouldDeferFetch) throws Exception * This sort of plan is never produced by the {@link com.apple.foundationdb.record.query.plan.RecordQueryPlanner}, * so we have to test the visitor directly. */ - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - public void intersectionVisitorOnComplexComparisonKey(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + @Test + public void intersectionVisitorOnComplexComparisonKey() throws Exception { complexQuerySetup(null); IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); RecordQueryPlan originalPlan1 = RecordQueryIntersectionPlan.from( - new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, serializationMode), - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, serializationMode), + new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false), primaryKey("MySimpleRecord")); RecordQueryPlan modifiedPlan1 = RecordQueryPlannerSubstitutionVisitor.applyRegularVisitors(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), originalPlan1, recordStore.getRecordMetaData(), PlannableIndexTypes.DEFAULT, primaryKey("MySimpleRecord")); @@ -983,8 +982,8 @@ public void intersectionVisitorOnComplexComparisonKey(@Nonnull final KeyValueCur coveringIndexScan(indexScan("MySimpleRecord$str_value_indexed")), coveringIndexScan(indexScan("MySimpleRecord$num_value_3_indexed"))))); RecordQueryPlan originalPlan2 = RecordQueryIntersectionPlan.from( - new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, serializationMode), - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, serializationMode), + new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false), concat(field("num_value_2"), primaryKey("MySimpleRecord"))); RecordQueryPlan modifiedPlan2 = RecordQueryPlannerSubstitutionVisitor.applyRegularVisitors(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), originalPlan2, recordStore.getRecordMetaData(), PlannableIndexTypes.DEFAULT, primaryKey("MySimpleRecord")); // Visitor should not perform transformation because of comparison key on num_value_unique diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java index 779d3a175f..3d73c64460 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java @@ -2341,16 +2341,15 @@ void testOrQueryNoIndex(OrQueryParams orQueryParams) throws Exception { * This sort of plan is never produced by the {@link com.apple.foundationdb.record.query.plan.RecordQueryPlanner}, * so we have to test the visitor directly. */ - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - void unionVisitorOnComplexComparisonKey(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + @Test + void unionVisitorOnComplexComparisonKey() throws Exception { complexQuerySetup(null); final IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); RecordQueryPlan originalPlan1 = RecordQueryUnionPlan.from( - new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, serializationMode), - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, serializationMode), + new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false), primaryKey("MySimpleRecord"), true); RecordQueryPlan modifiedPlan1 = RecordQueryPlannerSubstitutionVisitor.applyRegularVisitors(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), originalPlan1, recordStore.getRecordMetaData(), PlannableIndexTypes.DEFAULT, primaryKey("MySimpleRecord")); @@ -2365,8 +2364,8 @@ void unionVisitorOnComplexComparisonKey(@Nonnull final KeyValueCursorBase.Serial assertMatchesExactly(modifiedPlan1, planMatcher); RecordQueryPlan originalPlan2 = RecordQueryUnionPlan.from( - new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false, serializationMode), - new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false, serializationMode), + new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false), + new RecordQueryIndexPlan("MySimpleRecord$num_value_3_indexed", fullValueScan, false), concat(field("num_value_2"), primaryKey("MySimpleRecord")), true); RecordQueryPlan modifiedPlan2 = RecordQueryPlannerSubstitutionVisitor.applyRegularVisitors(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), originalPlan2, recordStore.getRecordMetaData(), PlannableIndexTypes.DEFAULT, primaryKey("MySimpleRecord")); // Visitor should not perform transformation because of comparison key on num_value_unique diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java index abf7f307b3..004ab99a4e 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java @@ -55,10 +55,10 @@ public class QueryPlanStructuralInstrumentationTest { private static int VALUE = 4; - private RecordQueryPlan indexPlanEquals(String indexName, Object value, KeyValueCursorBase.SerializationMode serializationMode) { + private RecordQueryPlan indexPlanEquals(String indexName, Object value) { IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, value)), Collections.emptySet())); - return new RecordQueryIndexPlan(indexName, scan, false, serializationMode); + return new RecordQueryIndexPlan(indexName, scan, false); } private void assertNoIndexes(RecordQueryPlan plan) { @@ -71,12 +71,11 @@ private void assertUsesIndexes(RecordQueryPlan plan, Collection indexes) assertTrue(used.containsAll(indexes)); } - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - public void indexPlan(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { + @Test + public void indexPlan() { final String indexName = "an_index"; StoreTimer timer = new FDBStoreTimer(); - RecordQueryPlan plan = indexPlanEquals(indexName, VALUE, serializationMode); + RecordQueryPlan plan = indexPlanEquals(indexName, VALUE); plan.logPlanStructure(timer); assertUsesIndexes(plan, Lists.newArrayList(indexName)); @@ -93,13 +92,12 @@ public void fullScan() { assertEquals(timer.getCount(FDBStoreTimer.Counts.PLAN_SCAN), 1); } - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - public void in(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { + @Test + public void in() { final String indexName = "a_field"; final IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.ParameterComparison(Comparisons.Type.EQUALS, "another_field")), Collections.emptySet())); final RecordQueryPlan plan = new RecordQueryInValuesJoinPlan( - new RecordQueryIndexPlan(indexName, scan, false, serializationMode), + new RecordQueryIndexPlan(indexName, scan, false), "another_field", Bindings.Internal.IN, Arrays.asList(2, 4), @@ -113,12 +111,11 @@ public void in(@Nonnull final KeyValueCursorBase.SerializationMode serialization assertEquals(timer.getCount(FDBStoreTimer.Counts.PLAN_INDEX), 1); } - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - public void unionSameIndex(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { + @Test + public void unionSameIndex() { final RecordQueryPlan plan = RecordQueryUnionPlan.from( - indexPlanEquals("index_1", 2, serializationMode), - indexPlanEquals("index_1", 4, serializationMode), + indexPlanEquals("index_1", 2), + indexPlanEquals("index_1", 4), EmptyKeyExpression.EMPTY, false); assertUsesIndexes(plan, Lists.newArrayList("index_1")); @@ -129,12 +126,11 @@ public void unionSameIndex(@Nonnull final KeyValueCursorBase.SerializationMode s assertEquals(timer.getCount(FDBStoreTimer.Counts.PLAN_INDEX), 2); } - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - public void unionDifferentIndex(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) { + @Test + public void unionDifferentIndex() { final RecordQueryPlan plan = RecordQueryUnionPlan.from( - indexPlanEquals("index_1", 2, serializationMode), - indexPlanEquals("index_2", 4, serializationMode), + indexPlanEquals("index_1", 2), + indexPlanEquals("index_2", 4), EmptyKeyExpression.EMPTY, false); assertUsesIndexes(plan, Lists.newArrayList("index_1", "index_2")); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/matchers/ExpressionMatcherTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/matchers/ExpressionMatcherTest.java index 7bade28bfe..3d0a350518 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/matchers/ExpressionMatcherTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/matchers/ExpressionMatcherTest.java @@ -78,7 +78,7 @@ public class ExpressionMatcherTest { RecordQueryPlanMatchers.indexPlan(), RelationalExpressionMatchers.ofType(RecordQueryPlan.class)); private static final List existingBindables = ImmutableList.of( - new RecordQueryIndexPlan("fake_index", IndexScanComparisons.byValue(), false, KeyValueCursorBase.SerializationMode.TO_OLD), + new RecordQueryIndexPlan("fake_index", IndexScanComparisons.byValue(), false), new RecordQueryScanPlan(ScanComparisons.EMPTY, false)); @Nonnull @@ -123,7 +123,7 @@ public void singleTypeMatcher() { BindingMatcher matcher = RecordQueryPlanMatchers.indexPlan(); final IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); final Reference root = - Reference.plannedOf(new RecordQueryIndexPlan("an_index", fullValueScan, true, KeyValueCursorBase.SerializationMode.TO_OLD)); + Reference.plannedOf(new RecordQueryIndexPlan("an_index", fullValueScan, true)); Optional newBindings = matcher.bindMatches(RecordQueryPlannerConfiguration.defaultPlannerConfiguration(), PlannerBindings.empty(), root.get()).findFirst(); // check the bindings are what we expect, and that none of the existing ones were clobbered assertTrue(newBindings.isPresent()); @@ -143,7 +143,7 @@ public void nestedTypeMatchers() { ListMatcher.exactly(QuantifierMatchers.physicalQuantifier(childMatcher1), QuantifierMatchers.physicalQuantifier(childMatcher2))); IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); - RecordQueryIndexPlan child1 = new RecordQueryIndexPlan("an_index", fullValueScan, true, KeyValueCursorBase.SerializationMode.TO_OLD); + RecordQueryIndexPlan child1 = new RecordQueryIndexPlan("an_index", fullValueScan, true); RecordQueryScanPlan child2 = new RecordQueryScanPlan(ScanComparisons.EMPTY, true); // check matches if the children are in the right order @@ -170,7 +170,7 @@ public void matchChildOrder() { QuantifierMatchers.physicalQuantifier(RecordQueryPlanMatchers.scanPlan()))); IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); - RecordQueryIndexPlan child1 = new RecordQueryIndexPlan("an_index", fullValueScan, true, KeyValueCursorBase.SerializationMode.TO_OLD); + RecordQueryIndexPlan child1 = new RecordQueryIndexPlan("an_index", fullValueScan, true); RecordQueryScanPlan child2 = new RecordQueryScanPlan(ScanComparisons.EMPTY, true); RelationalExpression root = RecordQueryUnionPlan.from( // union with arbitrary comparison key child1, child2, EmptyKeyExpression.EMPTY, false); @@ -190,7 +190,7 @@ public void matchChildrenAsReferences() { QuantifierMatchers.physicalQuantifierOverRef(childMatcher2))); IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); - RecordQueryIndexPlan child1 = new RecordQueryIndexPlan("an_index", fullValueScan, true, KeyValueCursorBase.SerializationMode.TO_OLD); + RecordQueryIndexPlan child1 = new RecordQueryIndexPlan("an_index", fullValueScan, true); RecordQueryScanPlan child2 = new RecordQueryScanPlan(ScanComparisons.EMPTY, true); RelationalExpression root = RecordQueryUnionPlan.from( // union with arbitrary comparison key child1, child2, EmptyKeyExpression.EMPTY, false); @@ -235,8 +235,7 @@ public void treeDescentWithMixedBindings() { Type.Record.fromFields(false, ImmutableList.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.INT), Optional.of("field1")), Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("field2")))), - QueryPlanConstraint.tautology(), - KeyValueCursorBase.SerializationMode.TO_OLD)); + QueryPlanConstraint.tautology())); final Quantifier.ForEach quantifier = Quantifier.forEach(baseRef); LogicalFilterExpression filterPlan = new LogicalFilterExpression(Query.and(andBranch1, andBranch2).expand(quantifier, () -> Quantifier.forEach(baseRef)).getPredicates(), diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitorTest.java index acf5b91cb7..39d749baf5 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitorTest.java @@ -221,7 +221,7 @@ private static NonnullPair randomIndexDetails(Random r) IndexScanParameters scanParameters = IndexScanComparisons.byValue(comparisons.getLeft(), scanType); String indexName = randomIndexName(r); boolean reverse = r.nextBoolean(); - return NonnullPair.of(new RecordQueryIndexPlan(indexName, scanParameters, reverse, KeyValueCursorBase.SerializationMode.TO_OLD), + return NonnullPair.of(new RecordQueryIndexPlan(indexName, scanParameters, reverse), String.format("%s %s%s%s", indexName, comparisons.getRight(), scanType == IndexScanType.BY_VALUE ? "" : (" " + scanType), reverse ? " REVERSE" : "")); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/planning/RecordQueryPlanEqualityTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/planning/RecordQueryPlanEqualityTest.java index 911a8bcbce..7102ba7146 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/planning/RecordQueryPlanEqualityTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/planning/RecordQueryPlanEqualityTest.java @@ -64,7 +64,7 @@ private RecordQueryPlan scanPlan() { private RecordQueryPlan indexPlanEquals(String indexName, Object value) { IndexScanParameters scan = IndexScanComparisons.byValue(new ScanComparisons(Arrays.asList(new Comparisons.SimpleComparison(Comparisons.Type.EQUALS, value)), Collections.emptySet())); - return new RecordQueryIndexPlan(indexName, scan, false, KeyValueCursorBase.SerializationMode.TO_OLD); + return new RecordQueryIndexPlan(indexName, scan, false); } private RecordQueryPlan unionPlan(Object value1, Object value2) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java index 9676e9ff06..e9a3f63afb 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java @@ -67,9 +67,8 @@ void simpleFieldValueTest() throws Exception { Assertions.assertEquals(fieldValue, parsedValue); } - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - void simpleIndexScanTest(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + @Test + void simpleIndexScanTest() throws Exception { final RecordQueryIndexPlan plan = new RecordQueryIndexPlan("an_index", null, IndexScanComparisons.byValue(), @@ -81,8 +80,7 @@ void simpleIndexScanTest(@Nonnull final KeyValueCursorBase.SerializationMode ser Type.Record.fromFields(false, ImmutableList.of(Type.Record.Field.of(Type.primitiveType(Type.TypeCode.INT), Optional.of("field1")), Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("field2")))), - QueryPlanConstraint.tautology(), - serializationMode); + QueryPlanConstraint.tautology()); PlanSerializationContext planSerializationContext = PlanSerializationContext.newForCurrentMode(); final PPlanReference proto = planSerializationContext.toPlanReferenceProto(plan); final byte[] bytes = proto.toByteArray(); @@ -93,9 +91,8 @@ void simpleIndexScanTest(@Nonnull final KeyValueCursorBase.SerializationMode ser Assertions.assertTrue(plan.semanticEquals(parsedPlan)); } - @ParameterizedTest - @EnumSource(KeyValueCursorBase.SerializationMode.class) - void recordQueryDefaultOnEmptyPlanTest(@Nonnull final KeyValueCursorBase.SerializationMode serializationMode) throws Exception { + @Test + void recordQueryDefaultOnEmptyPlanTest() throws Exception { final RecordQueryIndexPlan indexPlan = new RecordQueryIndexPlan("an_index", null, IndexScanComparisons.byValue(), @@ -105,8 +102,7 @@ void recordQueryDefaultOnEmptyPlanTest(@Nonnull final KeyValueCursorBase.Seriali false, Optional.empty(), Type.primitiveType(Type.TypeCode.INT, true), - QueryPlanConstraint.tautology(), - serializationMode); + QueryPlanConstraint.tautology()); final var plan = new RecordQueryDefaultOnEmptyPlan(Quantifier .Physical.physicalBuilder().withAlias(CorrelationIdentifier.of("q42")).build( Reference.plannedOf(indexPlan) From cc9567baf52933cfc7c2da1def282cb553b7c3a8 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Mon, 16 Jun 2025 20:32:53 -0700 Subject: [PATCH 13/15] add test --- .../query/ForceContinuationQueryTests.java | 96 ++++++++++++++++++- 1 file changed, 92 insertions(+), 4 deletions(-) diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ForceContinuationQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ForceContinuationQueryTests.java index ba1e5b758b..7fed79a85a 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ForceContinuationQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/ForceContinuationQueryTests.java @@ -34,6 +34,7 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -56,6 +57,7 @@ public class ForceContinuationQueryTests { "create index mv4 as select count(col1) from t1 group by col2\n" + "create table t2(id bigint, col1 bigint, col2 bigint, col3 bigint, primary key(id))\n" + + "create index mv5 as select col2 from t2\n" + "create index mv7 as select min_ever(col3) from t2\n" + "create table t3(id bigint, col1 bigint, col2 bigint, primary key(id))\n" + @@ -97,8 +99,8 @@ void setup() throws Exception { } @ParameterizedTest - @MethodSource("provideArguments") - void testOldSerialization(String sql, long result) throws Exception { + @MethodSource("failedQueries") + void testOldSerializationFails(String sql, long result) throws Exception { Continuation continuation; statement.setMaxRows(1); try (var resultSet = statement.executeQuery(sql)) { @@ -115,7 +117,7 @@ void testOldSerialization(String sql, long result) throws Exception { } @ParameterizedTest - @MethodSource("provideArguments") + @MethodSource("failedQueries") void testNewSerialization(String sql, long result) throws Exception { connection.setOption(Options.Name.KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW, true); Continuation continuation; @@ -137,8 +139,94 @@ void testNewSerialization(String sql, long result) throws Exception { } } + @Test + void testOldSerializationWorks() throws Exception { + Continuation continuation; + statement.setMaxRows(1); + try (var resultSet = statement.executeQuery("select count(*) from t2 group by col2")) { + Assertions.assertTrue(resultSet.next()); + Assertions.assertEquals(1L, resultSet.getLong(1)); + continuation = resultSet.getContinuation(); + } + try (final var preparedStatement = connection.prepareStatement("EXECUTE CONTINUATION ?param")) { + preparedStatement.setMaxRows(1); + preparedStatement.setBytes("param", continuation.serialize()); + try (var resultSet = preparedStatement.executeQuery()) { + Assertions.assertTrue(resultSet.next()); + Assertions.assertEquals(3L, resultSet.getLong(1)); + continuation = resultSet.getContinuation(); + } + } + try (final var preparedStatement = connection.prepareStatement("EXECUTE CONTINUATION ?param")) { + preparedStatement.setMaxRows(1); + preparedStatement.setBytes("param", continuation.serialize()); + try (var resultSet = preparedStatement.executeQuery()) { + Assertions.assertFalse(resultSet.next()); + } + } + } + + @Test + void testNewSerializationWorks() throws Exception { + connection.setOption(Options.Name.KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW, true); + Continuation continuation; + try (final var s = connection.createStatement()) { + s.setMaxRows(1); + try (var resultSet = s.executeQuery("select count(*) from t2 group by col2")) { + Assertions.assertTrue(resultSet.next()); + Assertions.assertEquals(1L, resultSet.getLong(1)); + continuation = resultSet.getContinuation(); + } + } + try (final var preparedStatement = connection.prepareStatement("EXECUTE CONTINUATION ?param")) { + preparedStatement.setMaxRows(1); + preparedStatement.setBytes("param", continuation.serialize()); + try (var resultSet = preparedStatement.executeQuery()) { + Assertions.assertTrue(resultSet.next()); + Assertions.assertEquals(3L, resultSet.getLong(1)); + continuation = resultSet.getContinuation(); + } + } + try (final var preparedStatement = connection.prepareStatement("EXECUTE CONTINUATION ?param")) { + preparedStatement.setMaxRows(1); + preparedStatement.setBytes("param", continuation.serialize()); + try (var resultSet = preparedStatement.executeQuery()) { + Assertions.assertFalse(resultSet.next()); + } + } + } + + @Test + void testOldThenNewWorks() throws Exception { + Continuation continuation; + try (final var s = connection.createStatement()) { + s.setMaxRows(1); + try (var resultSet = s.executeQuery("select count(*) from t2 group by col2")) { + Assertions.assertTrue(resultSet.next()); + Assertions.assertEquals(1L, resultSet.getLong(1)); + continuation = resultSet.getContinuation(); + } + } + connection.setOption(Options.Name.KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW, true); + try (final var preparedStatement = connection.prepareStatement("EXECUTE CONTINUATION ?param")) { + preparedStatement.setMaxRows(1); + preparedStatement.setBytes("param", continuation.serialize()); + try (var resultSet = preparedStatement.executeQuery()) { + Assertions.assertTrue(resultSet.next()); + Assertions.assertEquals(3L, resultSet.getLong(1)); + continuation = resultSet.getContinuation(); + } + } + try (final var preparedStatement = connection.prepareStatement("EXECUTE CONTINUATION ?param")) { + preparedStatement.setMaxRows(1); + preparedStatement.setBytes("param", continuation.serialize()); + try (var resultSet = preparedStatement.executeQuery()) { + Assertions.assertFalse(resultSet.next()); + } + } + } - private static Stream provideArguments() { + private static Stream failedQueries() { return Stream.of( // aggregate-index-count.yamsql Arguments.of("select count(*) from t1", 4L), From 09907fe0e49295ad26f0928b0a6577f483732301 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Mon, 16 Jun 2025 20:53:41 -0700 Subject: [PATCH 14/15] style --- .../foundationdb/KeyValueCursorBase.java | 5 -- .../plan/plans/RecordQueryIndexPlan.java | 2 - .../main/proto/record_planner_config.proto | 6 -- .../MultidimensionalIndexTestBase.java | 3 +- .../SimpleMultidimensionalIndexTest.java | 1 - .../SlowMultidimensionalIndexTest.java | 4 +- .../limits/FDBRecordStoreLimitTestBase.java | 1 - .../limits/FDBRecordStoreScanLimitTest.java | 5 -- .../query/FDBAndQueryToIntersectionTest.java | 3 - .../query/FDBOrQueryToUnionTest.java | 2 - ...ueryPlanStructuralInstrumentationTest.java | 4 -- .../matchers/ExpressionMatcherTest.java | 1 - .../plan/explain/ExplainPlanVisitorTest.java | 1 - .../planning/RecordQueryPlanEqualityTest.java | 1 - .../serialization/PlanSerializationTest.java | 4 -- .../recordlayer/query/GroupByQueryTests.java | 59 ------------------- .../resources/aggregate-index-tests.yamsql | 16 ++--- 17 files changed, 6 insertions(+), 112 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java index d9869e8a5d..1d2bdeb72f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/provider/foundationdb/KeyValueCursorBase.java @@ -28,7 +28,6 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; import com.apple.foundationdb.async.AsyncIterator; -import com.apple.foundationdb.record.ByteArrayContinuation; import com.apple.foundationdb.record.CursorStreamingMode; import com.apple.foundationdb.record.EndpointType; import com.apple.foundationdb.record.KeyRange; @@ -41,10 +40,7 @@ import com.apple.foundationdb.record.cursors.AsyncIteratorCursor; import com.apple.foundationdb.record.cursors.BaseCursor; import com.apple.foundationdb.record.cursors.CursorLimitManager; -import com.apple.foundationdb.record.cursors.aggregate.AggregateCursor; -import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; import com.apple.foundationdb.subspace.Subspace; -import com.apple.foundationdb.tuple.ByteArrayUtil2; import com.apple.foundationdb.tuple.Tuple; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; @@ -52,7 +48,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.awt.RenderingHints; import java.util.Arrays; import java.util.Optional; import java.util.concurrent.CompletableFuture; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java index 7e1811009b..066bff622a 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryIndexPlan.java @@ -37,7 +37,6 @@ import com.apple.foundationdb.record.RecordCursor; import com.apple.foundationdb.record.RecordCursorContinuation; import com.apple.foundationdb.record.RecordCursorEndContinuation; -import com.apple.foundationdb.record.RecordCursorProto; import com.apple.foundationdb.record.RecordCursorResult; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.ScanProperties; @@ -91,7 +90,6 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.protobuf.ByteString; -import com.google.protobuf.InvalidProtocolBufferException; import com.google.protobuf.Message; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/fdb-record-layer-core/src/main/proto/record_planner_config.proto b/fdb-record-layer-core/src/main/proto/record_planner_config.proto index 7156d9ff10..23b1a4f029 100644 --- a/fdb-record-layer-core/src/main/proto/record_planner_config.proto +++ b/fdb-record-layer-core/src/main/proto/record_planner_config.proto @@ -36,11 +36,6 @@ message PlannerConfiguration { USE_REMOTE_FETCH_WITH_FALLBACK = 2; } - enum KeyValueCursorContinuationSerializationMode { - TO_OLD = 0; - TO_NEW = 1; - } - message SortConfiguration { optional bool shouldAllowNonIndexSort = 1; } @@ -59,5 +54,4 @@ message PlannerConfiguration { optional int32 maxNumReplansForInToJoin = 12; optional int32 orToUnionMaxNumConjuncts = 13; optional int32 maxNumReplansForInUnion = 14; - optional KeyValueCursorContinuationSerializationMode keyValueCursorContinuationSerializationMode = 15; } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java index 13e905f357..c4ebd0b696 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/MultidimensionalIndexTestBase.java @@ -47,7 +47,6 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.provider.foundationdb.IndexScanBounds; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.provider.foundationdb.MultidimensionalIndexScanBounds; import com.apple.foundationdb.record.provider.foundationdb.query.FDBRecordStoreQueryTestBase; import com.apple.foundationdb.record.query.IndexQueryabilityFilter; @@ -497,7 +496,7 @@ void basicReadWithNullsTest(final boolean useAsync, @Nonnull final String storag } void indexReadTest(final boolean useAsync, final long seed, final int numRecords, @Nonnull final String storage, - final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { + final boolean storeHilbertValues, final boolean useNodeSlotIndex) throws Exception { final RecordMetaDataHook additionalIndexes = metaDataBuilder -> { addCalendarNameStartEpochIndex(metaDataBuilder); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java index cfe1bfa7be..f203aa1d04 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SimpleMultidimensionalIndexTest.java @@ -20,7 +20,6 @@ package com.apple.foundationdb.record.provider.foundationdb.indexes; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java index 366e84440c..2ca83d6c6b 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/indexes/SlowMultidimensionalIndexTest.java @@ -24,9 +24,7 @@ import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ExecuteProperties; import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; -import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; import com.apple.test.Tags; import com.google.common.collect.ImmutableList; import com.google.protobuf.Message; @@ -110,7 +108,7 @@ static Stream argumentsForIndexReadsAfterDeletes() { Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_NODE.toString(), false, true), Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_NODE.toString(), true, false), Arguments.of(random.nextLong(), 5000, random.nextInt(5000) + 1, BY_NODE.toString(), true, true) - )); + )); } static Stream argumentsForIndexReadsWithDuplicates() { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreLimitTestBase.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreLimitTestBase.java index 532cd50c80..af5642bfec 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreLimitTestBase.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreLimitTestBase.java @@ -26,7 +26,6 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreTestBase; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.expressions.Query; import com.apple.foundationdb.record.query.expressions.QueryComponent; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java index ab58e00731..23be321b10 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/limits/FDBRecordStoreScanLimitTest.java @@ -44,7 +44,6 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBStoredRecord; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.provider.foundationdb.SplitHelper; import com.apple.foundationdb.record.provider.foundationdb.cursors.ProbableIntersectionCursor; import com.apple.foundationdb.record.query.RecordQuery; @@ -351,10 +350,6 @@ void testSplitContinuation() { @ParameterizedTest @ValueSource(ints = {2, 5, 10, 20}) // for this test, the scan limit must divide 100 public void testExecuteStateReset(int scanLimit) throws Exception { - executeStateReset(scanLimit); - } - - private void executeStateReset(int scanLimit) throws Exception { final IndexScanParameters fullValueScan = IndexScanComparisons.byValue(); final RecordQueryPlan plan = new RecordQueryIndexPlan("MySimpleRecord$str_value_indexed", fullValueScan, false); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java index f3147fbdc3..c5367a6e87 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBAndQueryToIntersectionTest.java @@ -31,7 +31,6 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.expressions.Query; @@ -54,9 +53,7 @@ import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import javax.annotation.Nonnull; import java.util.Arrays; import java.util.Objects; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java index 3d73c64460..c03b44329d 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/query/FDBOrQueryToUnionTest.java @@ -40,7 +40,6 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordContext; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.RecordQuery; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.expressions.OrComponent; @@ -74,7 +73,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.EnumSource; import org.junit.jupiter.params.provider.MethodSource; import javax.annotation.Nonnull; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java index 004ab99a4e..a87dfac728 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/QueryPlanStructuralInstrumentationTest.java @@ -26,7 +26,6 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBStoreTimer; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInValuesJoinPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; @@ -36,10 +35,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryUnionPlan; import com.google.common.collect.Lists; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import javax.annotation.Nonnull; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/matchers/ExpressionMatcherTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/matchers/ExpressionMatcherTest.java index 3d0a350518..8a86564bae 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/matchers/ExpressionMatcherTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/matchers/ExpressionMatcherTest.java @@ -24,7 +24,6 @@ import com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.expressions.Query; import com.apple.foundationdb.record.query.expressions.QueryComponent; import com.apple.foundationdb.record.query.plan.QueryPlanConstraint; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitorTest.java index 39d749baf5..6142e28925 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitorTest.java @@ -38,7 +38,6 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.expressions.Query; import com.apple.foundationdb.record.query.expressions.QueryComponent; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/planning/RecordQueryPlanEqualityTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/planning/RecordQueryPlanEqualityTest.java index 7102ba7146..7040ef9f37 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/planning/RecordQueryPlanEqualityTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/planning/RecordQueryPlanEqualityTest.java @@ -24,7 +24,6 @@ import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; import com.apple.foundationdb.record.provider.foundationdb.IndexScanParameters; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.expressions.Comparisons; import com.apple.foundationdb.record.query.expressions.FieldWithComparison; import com.apple.foundationdb.record.query.expressions.Query; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java index e9a3f63afb..99443eaee6 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/serialization/PlanSerializationTest.java @@ -25,7 +25,6 @@ import com.apple.foundationdb.record.planprotos.PPlanReference; import com.apple.foundationdb.record.planprotos.PValue; import com.apple.foundationdb.record.provider.foundationdb.IndexScanComparisons; -import com.apple.foundationdb.record.provider.foundationdb.KeyValueCursorBase; import com.apple.foundationdb.record.query.plan.QueryPlanConstraint; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; @@ -43,10 +42,7 @@ import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; -import javax.annotation.Nonnull; import java.util.Optional; /** diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java index c1bd38ffe3..ba20d55cc1 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/GroupByQueryTests.java @@ -114,65 +114,6 @@ void groupByWithScanLimit() throws Exception { } } - @Test - void groupByForceContinuation() throws Exception { - final String schemaTemplate = - "create table t1(pk bigint, a bigint, b bigint, c bigint, primary key(pk))\n" + - "create index mv1 as select count(*) from t1\n" + - "create index mv2 as select b from t1"; - Continuation continuation = null; - try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { - try (var conn = ddl.setSchemaAndGetConnection()) { - conn.setOption(Options.Name.MAX_ROWS, 1); - conn.setOption(Options.Name.CONTINUATIONS_CONTAIN_COMPILED_STATEMENTS, true); - conn.setOption(Options.Name.KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW, false); - try (var statement = conn.createStatement()) { - insertT1Record(statement, 1, 10, 1, 20); - insertT1Record(statement, 2, 10, 2, 20); - insertT1Record(statement, 3, 10, 2, 5); - insertT1Record(statement, 4, 10, 2, 15); - insertT1Record(statement, 5, 10, 2, 5); - insertT1Record(statement, 6, 20, 3, 10); - insertT1Record(statement, 7, 20, 4, 40); - insertT1Record(statement, 8, 20, 4, 20); - insertT1Record(statement, 9, 20, 4, 90); - insertT1Record(statement, 10, 20, 4, 10); - insertT1Record(statement, 11, 20, 4, 40); - insertT1Record(statement, 12, 20, 4, 20); - insertT1Record(statement, 13, 20, 4, 90); - } - try (var statement = conn.createStatement()) { - String query = "select count(*) from t1 group by b"; - Assertions.assertTrue(statement.execute(query), "Did not return a result set from a select statement!"); - try (final RelationalResultSet resultSet = statement.getResultSet()) { - ResultSetAssert.assertThat(resultSet) - .hasNextRow() - .isRowExactly(1L) - .hasNoNextRow(); - continuation = resultSet.getContinuation(); - } - } - } - } - try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { - try (var conn = ddl.setSchemaAndGetConnection()) { - conn.setOption(Options.Name.MAX_ROWS, 1); - conn.setOption(Options.Name.CONTINUATIONS_CONTAIN_COMPILED_STATEMENTS, true); - conn.setOption(Options.Name.KEYVALUE_CURSOR_CONTINUATION_SERIALIZE_TO_NEW, false); - try (var statement = conn.prepareStatement("EXECUTE CONTINUATION ?param")) { - statement.setMaxRows(1); - statement.setBytes("param", continuation.serialize()); - try (final RelationalResultSet resultSet = statement.executeQuery()) { - ResultSetAssert.assertThat(resultSet) - .hasNoNextRow(); - continuation = resultSet.getContinuation(); - // Assertions.assertTrue(continuation.getExecutionState().length == 0); - } - } - } - } - } - @Test void groupByWithRowLimit() throws Exception { final String schemaTemplate = diff --git a/yaml-tests/src/test/resources/aggregate-index-tests.yamsql b/yaml-tests/src/test/resources/aggregate-index-tests.yamsql index f4b1e9ed39..6a8c01f3ee 100644 --- a/yaml-tests/src/test/resources/aggregate-index-tests.yamsql +++ b/yaml-tests/src/test/resources/aggregate-index-tests.yamsql @@ -281,12 +281,8 @@ test_block: # Cannot use FORCE_CONTINUATIONS with older versions due to: https://github.com/FoundationDB/fdb-record-layer/issues/3096 # (Extra values being produced after exhausting source of an aggregate cursor) # Can remove once we do not care about backwards compatibility before 4.1.9.0 - - maxRows: 1 - - result: [{COL3: 1, S: 1}] - - result: [{COL3: 2, S: 2}] - - result: [{COL3: 100, S: 1}] - - result: [{COL3: 200, S: 2}] - - result: [] + - maxRows: 0 + - result: [{COL3: 1, S: 1}, {COL3: 2, S: 2}, {COL3: 100, S: 1}, {COL3: 200, S: 2}] - - query: select col3, sum(col2) as s from t2 use index (mv9) where col1 = 1 group by col1, col3 order by col3 desc; - supported_version: 4.1.9.0 @@ -297,12 +293,8 @@ test_block: # Cannot use FORCE_CONTINUATIONS with older versions due to: https://github.com/FoundationDB/fdb-record-layer/issues/3096 # (Extra values being produced after exhausting source of an aggregate cursor) # Can remove once we do not care about backwards compatibility before 4.1.9.0 - - maxRows: 1 - - result: [{COL3: 200, S: 2}] - - result: [{COL3: 100, S: 1}] - - result: [{COL3: 2, S: 2}] - - result: [{COL3: 1, S: 1}] - - result: [] + - maxRows: 0 + - result: [{COL3: 200, S: 2}, {COL3: 100, S: 1}, {COL3: 2, S: 2}, {COL3: 1, S: 1}] # - # # grouping by constant is not yet supported. # - query: select sum(col2) from t1 group by 3,2,1; From 8343661d2f7aa66ebc175e227cb539f327c8f438 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Mon, 16 Jun 2025 22:29:10 -0700 Subject: [PATCH 15/15] checkstyle --- .../com/apple/foundationdb/record/ExecuteProperties.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/ExecuteProperties.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/ExecuteProperties.java index c3c9d60fe1..a16cae9209 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/ExecuteProperties.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/ExecuteProperties.java @@ -119,8 +119,9 @@ public ExecuteProperties setDryRun(final boolean isDryRun) { return copy(skip, rowLimit, timeLimit, isolationLevel, state, failOnScanLimitReached, defaultCursorStreamingMode, isDryRun, kvCursorContSerializeToNew); } - public boolean isKvCursorContSerializeToNew() {return kvCursorContSerializeToNew;} - + public boolean isKvCursorContSerializeToNew() { + return kvCursorContSerializeToNew; + } /** * Get the limit on the number of rows that will be returned as it would be passed to FDB.