From 3a5a44ba2106c40132d9281f119c8973b89f800f Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Fri, 28 Jul 2023 13:19:21 -0700 Subject: [PATCH 01/41] save --- .../record/provider/foundationdb/FDBMetaDataStoreTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java index b9f73b187c..32ec21b19f 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java @@ -187,6 +187,7 @@ public void manyTypes() { .setFanType(RecordMetaDataProto.Field.FanType.SCALAR) .setFieldName("field_1"); } + System.out.println("metaDataProto:" + metaData.build()); metaDataStore.saveRecordMetaData(metaData.build()); @@ -265,6 +266,7 @@ public void historyCompat() { .setName("MyOtherRecord") .getPrimaryKeyBuilder().getFieldBuilder().setFieldName("rec_no").setFanType(RecordMetaDataProto.Field.FanType.SCALAR); metaData.setVersion(102); + System.out.println("index metadata:" + metaData.build()); metaDataStore.saveRecordMetaData(metaData.build()); context.commit(); } From 53416bf6cb2fc167db93c11c656056c9d911ea34 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Fri, 28 Jul 2023 13:20:15 -0700 Subject: [PATCH 02/41] Revert "save" This reverts commit 3a5a44ba2106c40132d9281f119c8973b89f800f. --- .../record/provider/foundationdb/FDBMetaDataStoreTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java index 32ec21b19f..b9f73b187c 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java @@ -187,7 +187,6 @@ public void manyTypes() { .setFanType(RecordMetaDataProto.Field.FanType.SCALAR) .setFieldName("field_1"); } - System.out.println("metaDataProto:" + metaData.build()); metaDataStore.saveRecordMetaData(metaData.build()); @@ -266,7 +265,6 @@ public void historyCompat() { .setName("MyOtherRecord") .getPrimaryKeyBuilder().getFieldBuilder().setFieldName("rec_no").setFanType(RecordMetaDataProto.Field.FanType.SCALAR); metaData.setVersion(102); - System.out.println("index metadata:" + metaData.build()); metaDataStore.saveRecordMetaData(metaData.build()); context.commit(); } From 651816c7aac4dda03535513dd87aa68c51fe9cf4 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 20 Nov 2024 15:15:57 -0800 Subject: [PATCH 03/41] save --- .../foundationdb/record/RecordMetaData.java | 11 ++++ .../record/RecordMetaDataBuilder.java | 18 ++++++- .../foundationdb/record/metadata/UDF.java | 53 +++++++++++++++++++ .../expressions/DimensionsKeyExpression.java | 2 +- .../expressions/EmptyKeyExpression.java | 2 +- .../expressions/FunctionKeyExpression.java | 2 +- .../expressions/GroupingKeyExpression.java | 2 +- .../metadata/expressions/KeyExpression.java | 2 +- .../expressions/KeyWithValueExpression.java | 2 +- .../expressions/ListKeyExpression.java | 2 +- .../expressions/LiteralKeyExpression.java | 2 +- .../expressions/NestingKeyExpression.java | 2 +- .../expressions/RecordTypeKeyExpression.java | 2 +- .../expressions/SplitKeyExpression.java | 2 +- .../expressions/ThenKeyExpression.java | 2 +- .../expressions/VersionKeyExpression.java | 2 +- .../src/main/proto/record_metadata.proto | 5 ++ .../record/metadata/UnknownKeyExpression.java | 2 +- 18 files changed, 100 insertions(+), 15 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java index 9bf803bc64..ede961c461 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java @@ -28,6 +28,7 @@ import com.apple.foundationdb.record.metadata.MetaDataException; import com.apple.foundationdb.record.metadata.RecordType; import com.apple.foundationdb.record.metadata.SyntheticRecordType; +import com.apple.foundationdb.record.metadata.UDF; import com.apple.foundationdb.record.metadata.UnnestedRecordType; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression; @@ -46,6 +47,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -83,6 +85,8 @@ public class RecordMetaData implements RecordMetaDataProvider { @Nonnull private final Map> recordTypeKeyToSyntheticTypeMap; @Nonnull + private final Map udfMap; + @Nonnull private final Map indexes; @Nonnull private final Map universalIndexes; @@ -112,6 +116,7 @@ protected RecordMetaData(@Nonnull RecordMetaData orig) { Collections.unmodifiableMap(orig.indexes), Collections.unmodifiableMap(orig.universalIndexes), Collections.unmodifiableList(orig.formerIndexes), + Collections.unmodifiableMap(orig.udfMap), orig.splitLongRecords, orig.storeRecordVersions, orig.version, @@ -131,6 +136,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor, @Nonnull Map indexes, @Nonnull Map universalIndexes, @Nonnull List formerIndexes, + @Nonnull Map udfMap, boolean splitLongRecords, boolean storeRecordVersions, int version, @@ -147,6 +153,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor, this.indexes = indexes; this.universalIndexes = universalIndexes; this.formerIndexes = formerIndexes; + this.udfMap = udfMap; this.splitLongRecords = splitLongRecords; this.storeRecordVersions = storeRecordVersions; this.version = version; @@ -342,6 +349,9 @@ public List getFormerIndexes() { return formerIndexes; } + @Nullable + public UDF getUDF(@Nonnull String name) {return udfMap.get(name);} + public boolean isSplitLongRecords() { return splitLongRecords; } @@ -693,6 +703,7 @@ public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor } // Add in the final options. + builder.addAllUdfs(udfMap.values().stream().map(UDF::toProto).collect(Collectors.toList())); builder.setSplitLongRecords(splitLongRecords); builder.setStoreRecordVersions(storeRecordVersions); builder.setVersion(version); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index acf1657dcf..c4200d267f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -35,6 +35,7 @@ import com.apple.foundationdb.record.metadata.RecordTypeIndexesBuilder; import com.apple.foundationdb.record.metadata.SyntheticRecordType; import com.apple.foundationdb.record.metadata.SyntheticRecordTypeBuilder; +import com.apple.foundationdb.record.metadata.UDF; import com.apple.foundationdb.record.metadata.UnnestedRecordTypeBuilder; import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; @@ -110,6 +111,8 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider { @Nonnull private final Map> syntheticRecordTypes; @Nonnull + private final Map udfMap; + @Nonnull private final Map indexes; @Nonnull private final Map universalIndexes; @@ -144,6 +147,7 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider { indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance(); evolutionValidator = MetaDataEvolutionValidator.getDefaultInstance(); syntheticRecordTypes = new HashMap<>(); + udfMap = new HashMap<>(); } private void processSchemaOptions(boolean processExtensionOptions) { @@ -222,6 +226,9 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey())); } } + for (RecordMetaDataProto.UDF udf: metaDataProto.getUdfsList()) { + udfMap.put(udf.getFunction().getName(), new UDF(udf.getFunction().getName(), KeyExpression.fromProto(udf.getFunction().getArguments()))); + } if (metaDataProto.hasSplitLongRecords()) { splitLongRecords = metaDataProto.getSplitLongRecords(); } @@ -1179,6 +1186,15 @@ public void addFormerIndex(@Nonnull FormerIndex formerIndex) { formerIndexes.add(formerIndex); } + public void addUDF(@Nonnull UDF udf) { + // (TODO): check + udfMap.put(udf.getUdfName(), udf); + } + + public void addUDFList(@Nonnull List udfList) { + udfList.forEach(this::addUDF); + } + public boolean isSplitLongRecords() { return splitLongRecords; } @@ -1420,7 +1436,7 @@ public RecordMetaData build(boolean validate) { Map> recordTypeKeyToSyntheticRecordTypeMap = Maps.newHashMapWithExpectedSize(syntheticRecordTypes.size()); RecordMetaData metaData = new RecordMetaData(recordsDescriptor, getUnionDescriptor(), unionFields, builtRecordTypes, builtSyntheticRecordTypes, recordTypeKeyToSyntheticRecordTypeMap, - indexes, universalIndexes, formerIndexes, + indexes, universalIndexes, formerIndexes, udfMap, splitLongRecords, storeRecordVersions, version, subspaceKeyCounter, usesSubspaceKeyCounter, recordCountKey, localFileDescriptor != null); for (RecordTypeBuilder recordTypeBuilder : recordTypes.values()) { KeyExpression primaryKey = recordTypeBuilder.getPrimaryKey(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java new file mode 100644 index 0000000000..7c5d5e242c --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java @@ -0,0 +1,53 @@ +/* + * UDF.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.metadata; + +import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression; + +import javax.annotation.Nonnull; + +public class UDF { + @Nonnull private final String udfName; + @Nonnull private final KeyExpression keyExpression; + + public UDF(@Nonnull String udfName, @Nonnull KeyExpression keyExpression) { + this.udfName = udfName; + this.keyExpression = keyExpression; + } + + @Nonnull + public String getUdfName() { + return udfName; + } + + @Nonnull + public KeyExpression getKeyExpression() { + return keyExpression; + } + + public RecordMetaDataProto.UDF toProto() { + return RecordMetaDataProto.UDF.newBuilder() + .setFunction(RecordMetaDataProto.Function.newBuilder() + .setName(udfName)) + .build(); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java index add3ac1dc5..27f0df36d1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java @@ -114,7 +114,7 @@ public boolean needsCopyingToPartialRecord() { @Nonnull @Override - public RecordMetaDataProto.Dimensions toProto() throws SerializationException { + public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { final RecordMetaDataProto.Dimensions.Builder builder = RecordMetaDataProto.Dimensions.newBuilder(); builder.setWholeKey(getWholeKey().toKeyExpression()); builder.setPrefixSize(prefixSize); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java index f02581d3af..e43194b66e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java @@ -75,7 +75,7 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.Empty toProto() throws SerializationException { + public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { return RecordMetaDataProto.Empty.getDefaultInstance(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java index 4ab23d4fae..bf29bcd209 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java @@ -242,7 +242,7 @@ public static FunctionKeyExpression fromProto(RecordMetaDataProto.Function funct @Nonnull @Override - public final RecordMetaDataProto.Function toProto() throws SerializationException { + public final RecordMetaDataProto.KeyExpression toProto() throws SerializationException { RecordMetaDataProto.Function.Builder builder = RecordMetaDataProto.Function.newBuilder() .setName(getName()); builder.setArguments(getArguments().toKeyExpression()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java index eebb1678db..77416d41c8 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java @@ -100,7 +100,7 @@ public boolean needsCopyingToPartialRecord() { @Nonnull @Override - public RecordMetaDataProto.Grouping toProto() throws SerializationException { + public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { final RecordMetaDataProto.Grouping.Builder builder = RecordMetaDataProto.Grouping.newBuilder(); builder.setWholeKey(getWholeKey().toKeyExpression()); builder.setGroupedCount(groupedCount); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java index ca3c01f0ef..540bb2314e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java @@ -201,7 +201,7 @@ public static FanType valueOf(RecordMetaDataProto.Field.FanType fanType) throws } @Nonnull - Message toProto() throws SerializationException; + RecordMetaDataProto.KeyExpression toProto() throws SerializationException; @Nonnull RecordMetaDataProto.KeyExpression toKeyExpression(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java index ea224bc67e..9d6e7ecf77 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java @@ -133,7 +133,7 @@ public boolean needsCopyingToPartialRecord() { @Nonnull @Override - public RecordMetaDataProto.KeyWithValue toProto() throws SerializationException { + public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { return RecordMetaDataProto.KeyWithValue.newBuilder() .setInnerKey(getInnerKey().toKeyExpression()) .setSplitPoint(splitPoint) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java index 8552719078..6846324ad8 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java @@ -140,7 +140,7 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.List toProto() throws SerializationException { + public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { final RecordMetaDataProto.List.Builder builder = RecordMetaDataProto.List.newBuilder(); for (KeyExpression child : children) { builder.addChild(child.toKeyExpression()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java index 883707b34f..c128cd29f1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java @@ -98,7 +98,7 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.Value toProto() throws SerializationException { + public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { return proto; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java index 6b0e823ae5..283b2a308a 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java @@ -100,7 +100,7 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.Nesting toProto() throws SerializationException { + public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { final RecordMetaDataProto.Nesting.Builder builder = RecordMetaDataProto.Nesting.newBuilder(); builder.setParent(parent.toProto()); builder.setChild(getChild().toKeyExpression()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java index 8bdb00eab7..ee24b9d59c 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java @@ -107,7 +107,7 @@ public GroupingKeyExpression ungrouped() { @Nonnull @Override - public RecordMetaDataProto.RecordTypeKey toProto() throws SerializationException { + public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { return RecordMetaDataProto.RecordTypeKey.getDefaultInstance(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java index a669faa522..709bd4bad3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java @@ -110,7 +110,7 @@ public boolean needsCopyingToPartialRecord() { @Nonnull @Override - public RecordMetaDataProto.Split toProto() throws SerializationException { + public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { final RecordMetaDataProto.Split.Builder builder = RecordMetaDataProto.Split.newBuilder(); builder.setJoined(getJoined().toKeyExpression()); builder.setSplitSize(splitSize); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java index 776c48632f..a327e1b0ef 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java @@ -171,7 +171,7 @@ public GroupingKeyExpression group(int count) { @Nonnull @Override - public RecordMetaDataProto.Then toProto() throws SerializationException { + public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { final RecordMetaDataProto.Then.Builder builder = RecordMetaDataProto.Then.newBuilder(); for (KeyExpression child : children) { builder.addChild(child.toKeyExpression()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java index 632f269cd7..c4bd6bb258 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java @@ -97,7 +97,7 @@ public GroupingKeyExpression groupBy(@Nonnull KeyExpression groupByFirst, @Nonnu @Nonnull @Override - public RecordMetaDataProto.Version toProto() throws SerializationException { + public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { return RecordMetaDataProto.Version.getDefaultInstance(); } diff --git a/fdb-record-layer-core/src/main/proto/record_metadata.proto b/fdb-record-layer-core/src/main/proto/record_metadata.proto index 29fbb26cc1..df69e6dca2 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata.proto @@ -97,9 +97,14 @@ message MetaData { optional bool uses_subspace_key_counter = 11; repeated JoinedRecordType joined_record_types = 12; repeated UnnestedRecordType unnested_record_types = 13; + repeated UDF udfs = 14; extensions 1000 to 2000; } +message UDF { + optional Function function = 1; +} + message Then { repeated KeyExpression child = 1; } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java index 395cbca5b7..e241e1254d 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java @@ -81,7 +81,7 @@ public R expand(@Nonnull final KeyExpr @Nonnull @Override - public Message toProto() throws SerializationException { + public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { throw new UnsupportedOperationException("UnknownKeyExpressions cannot be converted to Protobuf"); } From 2c886c8110b8579c9becd359628fa2463e851964 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 5 Dec 2024 11:50:44 -0800 Subject: [PATCH 04/41] save --- .../java/com/apple/foundationdb/record/RecordMetaData.java | 3 +++ .../com/apple/foundationdb/record/RecordMetaDataBuilder.java | 3 ++- .../main/java/com/apple/foundationdb/record/metadata/UDF.java | 3 ++- .../record/metadata/expressions/DimensionsKeyExpression.java | 2 +- .../record/metadata/expressions/EmptyKeyExpression.java | 2 +- .../record/metadata/expressions/FunctionKeyExpression.java | 2 +- .../record/metadata/expressions/GroupingKeyExpression.java | 2 +- .../record/metadata/expressions/KeyExpression.java | 2 +- .../record/metadata/expressions/KeyWithValueExpression.java | 2 +- .../record/metadata/expressions/ListKeyExpression.java | 2 +- .../record/metadata/expressions/LiteralKeyExpression.java | 2 +- .../record/metadata/expressions/NestingKeyExpression.java | 2 +- .../record/metadata/expressions/RecordTypeKeyExpression.java | 2 +- .../record/metadata/expressions/SplitKeyExpression.java | 2 +- .../record/metadata/expressions/ThenKeyExpression.java | 2 +- .../record/metadata/expressions/VersionKeyExpression.java | 2 +- .../record/query/plan/cascades/BuiltInFunction.java | 2 +- .../record/query/plan/cascades/values/ArithmeticValue.java | 2 +- .../record/query/plan/cascades/values/FieldValue.java | 1 + .../foundationdb/record/metadata/UnknownKeyExpression.java | 2 +- 20 files changed, 24 insertions(+), 18 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java index ede961c461..35e3b9efe7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java @@ -352,6 +352,9 @@ public List getFormerIndexes() { @Nullable public UDF getUDF(@Nonnull String name) {return udfMap.get(name);} + @Nonnull + public Collection getAllUDFs() {return udfMap.values();} + public boolean isSplitLongRecords() { return splitLongRecords; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index c4200d267f..a374c848d6 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -54,6 +54,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -1191,7 +1192,7 @@ public void addUDF(@Nonnull UDF udf) { udfMap.put(udf.getUdfName(), udf); } - public void addUDFList(@Nonnull List udfList) { + public void addUDFs(@Nonnull Collection udfList) { udfList.forEach(this::addUDF); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java index 7c5d5e242c..dc891bae67 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java @@ -47,7 +47,8 @@ public KeyExpression getKeyExpression() { public RecordMetaDataProto.UDF toProto() { return RecordMetaDataProto.UDF.newBuilder() .setFunction(RecordMetaDataProto.Function.newBuilder() - .setName(udfName)) + .setName(udfName) + .setArguments(keyExpression.toKeyExpression())) .build(); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java index 27f0df36d1..add3ac1dc5 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java @@ -114,7 +114,7 @@ public boolean needsCopyingToPartialRecord() { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public RecordMetaDataProto.Dimensions toProto() throws SerializationException { final RecordMetaDataProto.Dimensions.Builder builder = RecordMetaDataProto.Dimensions.newBuilder(); builder.setWholeKey(getWholeKey().toKeyExpression()); builder.setPrefixSize(prefixSize); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java index e43194b66e..f02581d3af 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java @@ -75,7 +75,7 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public RecordMetaDataProto.Empty toProto() throws SerializationException { return RecordMetaDataProto.Empty.getDefaultInstance(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java index bf29bcd209..4ab23d4fae 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java @@ -242,7 +242,7 @@ public static FunctionKeyExpression fromProto(RecordMetaDataProto.Function funct @Nonnull @Override - public final RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public final RecordMetaDataProto.Function toProto() throws SerializationException { RecordMetaDataProto.Function.Builder builder = RecordMetaDataProto.Function.newBuilder() .setName(getName()); builder.setArguments(getArguments().toKeyExpression()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java index 77416d41c8..eebb1678db 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java @@ -100,7 +100,7 @@ public boolean needsCopyingToPartialRecord() { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public RecordMetaDataProto.Grouping toProto() throws SerializationException { final RecordMetaDataProto.Grouping.Builder builder = RecordMetaDataProto.Grouping.newBuilder(); builder.setWholeKey(getWholeKey().toKeyExpression()); builder.setGroupedCount(groupedCount); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java index 540bb2314e..ca3c01f0ef 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java @@ -201,7 +201,7 @@ public static FanType valueOf(RecordMetaDataProto.Field.FanType fanType) throws } @Nonnull - RecordMetaDataProto.KeyExpression toProto() throws SerializationException; + Message toProto() throws SerializationException; @Nonnull RecordMetaDataProto.KeyExpression toKeyExpression(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java index 9d6e7ecf77..ea224bc67e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java @@ -133,7 +133,7 @@ public boolean needsCopyingToPartialRecord() { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public RecordMetaDataProto.KeyWithValue toProto() throws SerializationException { return RecordMetaDataProto.KeyWithValue.newBuilder() .setInnerKey(getInnerKey().toKeyExpression()) .setSplitPoint(splitPoint) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java index 6846324ad8..8552719078 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java @@ -140,7 +140,7 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public RecordMetaDataProto.List toProto() throws SerializationException { final RecordMetaDataProto.List.Builder builder = RecordMetaDataProto.List.newBuilder(); for (KeyExpression child : children) { builder.addChild(child.toKeyExpression()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java index c128cd29f1..883707b34f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java @@ -98,7 +98,7 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public RecordMetaDataProto.Value toProto() throws SerializationException { return proto; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java index 283b2a308a..6b0e823ae5 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java @@ -100,7 +100,7 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public RecordMetaDataProto.Nesting toProto() throws SerializationException { final RecordMetaDataProto.Nesting.Builder builder = RecordMetaDataProto.Nesting.newBuilder(); builder.setParent(parent.toProto()); builder.setChild(getChild().toKeyExpression()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java index ee24b9d59c..8bdb00eab7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java @@ -107,7 +107,7 @@ public GroupingKeyExpression ungrouped() { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public RecordMetaDataProto.RecordTypeKey toProto() throws SerializationException { return RecordMetaDataProto.RecordTypeKey.getDefaultInstance(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java index 709bd4bad3..a669faa522 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java @@ -110,7 +110,7 @@ public boolean needsCopyingToPartialRecord() { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public RecordMetaDataProto.Split toProto() throws SerializationException { final RecordMetaDataProto.Split.Builder builder = RecordMetaDataProto.Split.newBuilder(); builder.setJoined(getJoined().toKeyExpression()); builder.setSplitSize(splitSize); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java index a327e1b0ef..776c48632f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java @@ -171,7 +171,7 @@ public GroupingKeyExpression group(int count) { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public RecordMetaDataProto.Then toProto() throws SerializationException { final RecordMetaDataProto.Then.Builder builder = RecordMetaDataProto.Then.newBuilder(); for (KeyExpression child : children) { builder.addChild(child.toKeyExpression()); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java index c4bd6bb258..632f269cd7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java @@ -97,7 +97,7 @@ public GroupingKeyExpression groupBy(@Nonnull KeyExpression groupByFirst, @Nonnu @Nonnull @Override - public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public RecordMetaDataProto.Version toProto() throws SerializationException { return RecordMetaDataProto.Version.getDefaultInstance(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java index 29b7d51817..500f1a8978 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java @@ -63,7 +63,7 @@ public abstract class BuiltInFunction { * @param parameterTypes The type of the parameter(s). * @param encapsulationFunction An encapsulation of the function's runtime computation. */ - protected BuiltInFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final EncapsulationFunction encapsulationFunction) { + public BuiltInFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final EncapsulationFunction encapsulationFunction) { this(functionName, parameterTypes, null, encapsulationFunction); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArithmeticValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArithmeticValue.java index 4c2e721f0e..d256d18ef9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArithmeticValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArithmeticValue.java @@ -200,7 +200,7 @@ private static Map, PhysicalOperator } @Nonnull - private static Value encapsulateInternal(@Nonnull BuiltInFunction builtInFunction, + public static Value encapsulateInternal(@Nonnull BuiltInFunction builtInFunction, @Nonnull final List arguments) { return encapsulate(builtInFunction.getFunctionName(), arguments); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FieldValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FieldValue.java index 84b90eeb27..48efd58dac 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FieldValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FieldValue.java @@ -257,6 +257,7 @@ public static FieldPath resolveFieldPath(@Nonnull final Type inputType, @Nonnull final Field field; final int ordinal; if (fieldName != null) { + System.out.println("FieldValue resolveFieldPath fieldNameFieldMap:" + fieldNameFieldMap.keySet() + " fieldName:" + fieldName); SemanticException.check(fieldNameFieldMap.containsKey(fieldName), SemanticException.ErrorCode.RECORD_DOES_NOT_CONTAIN_FIELD); field = fieldNameFieldMap.get(fieldName); final var fieldOrdinalsMap = Objects.requireNonNull(recordType.getFieldNameToOrdinalMap()); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java index e241e1254d..0190f42fcc 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java @@ -81,7 +81,7 @@ public R expand(@Nonnull final KeyExpr @Nonnull @Override - public RecordMetaDataProto.KeyExpression toProto() throws SerializationException { + public RecordMetaDataProto.Version toProto() throws SerializationException { throw new UnsupportedOperationException("UnknownKeyExpressions cannot be converted to Protobuf"); } From 38a800468d9fcf9976a038dcbab3fc551959b8b1 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 11 Dec 2024 13:36:03 -0800 Subject: [PATCH 05/41] save --- .../record/query/plan/cascades/values/ArithmeticValue.java | 2 +- .../record/query/plan/cascades/values/FieldValue.java | 1 - .../foundationdb/record/metadata/UnknownKeyExpression.java | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArithmeticValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArithmeticValue.java index d256d18ef9..4c2e721f0e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArithmeticValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/ArithmeticValue.java @@ -200,7 +200,7 @@ private static Map, PhysicalOperator } @Nonnull - public static Value encapsulateInternal(@Nonnull BuiltInFunction builtInFunction, + private static Value encapsulateInternal(@Nonnull BuiltInFunction builtInFunction, @Nonnull final List arguments) { return encapsulate(builtInFunction.getFunctionName(), arguments); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FieldValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FieldValue.java index 48efd58dac..84b90eeb27 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FieldValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FieldValue.java @@ -257,7 +257,6 @@ public static FieldPath resolveFieldPath(@Nonnull final Type inputType, @Nonnull final Field field; final int ordinal; if (fieldName != null) { - System.out.println("FieldValue resolveFieldPath fieldNameFieldMap:" + fieldNameFieldMap.keySet() + " fieldName:" + fieldName); SemanticException.check(fieldNameFieldMap.containsKey(fieldName), SemanticException.ErrorCode.RECORD_DOES_NOT_CONTAIN_FIELD); field = fieldNameFieldMap.get(fieldName); final var fieldOrdinalsMap = Objects.requireNonNull(recordType.getFieldNameToOrdinalMap()); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java index 0190f42fcc..395cbca5b7 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java @@ -81,7 +81,7 @@ public R expand(@Nonnull final KeyExpr @Nonnull @Override - public RecordMetaDataProto.Version toProto() throws SerializationException { + public Message toProto() throws SerializationException { throw new UnsupportedOperationException("UnknownKeyExpressions cannot be converted to Protobuf"); } From 4f7a23f9bd96129dc16458a44bf1558918970b81 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 11 Dec 2024 20:24:33 -0800 Subject: [PATCH 06/41] save --- .../record/RecordMetaDataBuilder.java | 6 +- .../foundationdb/record/metadata/UDF.java | 23 +- .../expressions/LiteralKeyExpression.java | 15 +- .../record/query/expressions/Comparisons.java | 2 +- .../plan/IndexKeyValueToPartialRecord.java | 2 +- .../src/main/proto/planner_debugger.proto | 3 - .../src/main/proto/record_metadata.proto | 25 +- .../main/proto/record_metadata_options.proto | 3 +- .../src/main/proto/record_query_plan.proto | 935 +---------------- .../src/main/proto/record_query_runtime.proto | 26 +- .../src/main/proto/record_value.proto | 962 ++++++++++++++++++ .../MetaDataEvolutionValidatorTest.java | 2 +- .../expressions/LiteralKeyExpressionTest.java | 4 +- 13 files changed, 1013 insertions(+), 995 deletions(-) create mode 100644 fdb-record-layer-core/src/main/proto/record_value.proto diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index a374c848d6..1efaee08b9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -43,6 +43,8 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; import com.apple.foundationdb.record.provider.foundationdb.MetaDataProtoEditor; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; import com.google.common.base.Verify; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -227,8 +229,10 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey())); } } + PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, + PlanHashable.CURRENT_FOR_CONTINUATION); for (RecordMetaDataProto.UDF udf: metaDataProto.getUdfsList()) { - udfMap.put(udf.getFunction().getName(), new UDF(udf.getFunction().getName(), KeyExpression.fromProto(udf.getFunction().getArguments()))); + udfMap.put(udf.getName(), new UDF(udf.getName(), Value.fromValueProto(serializationContext, udf.getValue()))); } if (metaDataProto.hasSplitLongRecords()) { splitLongRecords = metaDataProto.getSplitLongRecords(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java index dc891bae67..bb2410b1d3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java @@ -20,18 +20,22 @@ package com.apple.foundationdb.record.metadata; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; import com.apple.foundationdb.record.RecordMetaDataProto; -import com.apple.foundationdb.record.metadata.expressions.KeyExpression; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; import javax.annotation.Nonnull; public class UDF { @Nonnull private final String udfName; - @Nonnull private final KeyExpression keyExpression; + @Nonnull + private final Value value; - public UDF(@Nonnull String udfName, @Nonnull KeyExpression keyExpression) { + public UDF(@Nonnull String udfName, @Nonnull Value value) { this.udfName = udfName; - this.keyExpression = keyExpression; + this.value = value; } @Nonnull @@ -40,15 +44,14 @@ public String getUdfName() { } @Nonnull - public KeyExpression getKeyExpression() { - return keyExpression; - } + public Value getValue() {return value;} public RecordMetaDataProto.UDF toProto() { + PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, + PlanHashable.CURRENT_FOR_CONTINUATION); return RecordMetaDataProto.UDF.newBuilder() - .setFunction(RecordMetaDataProto.Function.newBuilder() - .setName(udfName) - .setArguments(keyExpression.toKeyExpression())) + .setName(udfName) + .setValue(value.toValueProto(serializationContext)) .build(); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java index 883707b34f..8cf2d8ea26 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java @@ -26,6 +26,7 @@ import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.Key; +import com.apple.foundationdb.record.planprotos.RecordMetaDataValueProto; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; @@ -57,14 +58,14 @@ public class LiteralKeyExpression extends BaseKeyExpression implements AtomKe @Nonnull private final List evaluated; @Nonnull - private final RecordMetaDataProto.Value proto; + private final com.apple.foundationdb.record.planprotos.Value proto; public LiteralKeyExpression(@Nullable T value) { // getProto() performs validation that it is a type we can serialize this(value, toProtoValue(value)); } - private LiteralKeyExpression(@Nullable T value, @Nonnull RecordMetaDataProto.Value proto) { + private LiteralKeyExpression(@Nullable T value, @Nonnull com.apple.foundationdb.record.planprotos.Value proto) { this.value = value; this.evaluated = ImmutableList.of(value == null ? Key.Evaluated.NULL : Key.Evaluated.scalar(value)); this.proto = proto; @@ -98,7 +99,7 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.Value toProto() throws SerializationException { + public com.apple.foundationdb.record.planprotos.Value toProto() throws SerializationException { return proto; } @@ -127,12 +128,12 @@ public boolean needsCopyingToPartialRecord() { } @Nonnull - public static LiteralKeyExpression fromProto(RecordMetaDataProto.Value proto) { + public static LiteralKeyExpression fromProto(com.apple.foundationdb.record.planprotos.Value proto) { return new LiteralKeyExpression<>(fromProtoValue(proto), proto); } @Nullable - public static Object fromProtoValue(RecordMetaDataProto.Value proto) { + public static Object fromProtoValue(com.apple.foundationdb.record.planprotos.Value proto) { int found = 0; Object value = null; if (proto.hasDoubleValue()) { @@ -174,8 +175,8 @@ public static Object fromProtoValue(RecordMetaDataProto.Value proto) { } @Nonnull - public static RecordMetaDataProto.Value toProtoValue(@Nullable Object value) { - RecordMetaDataProto.Value.Builder builder = RecordMetaDataProto.Value.newBuilder(); + public static com.apple.foundationdb.record.planprotos.Value toProtoValue(@Nullable Object value) { + com.apple.foundationdb.record.planprotos.Value.Builder builder = com.apple.foundationdb.record.planprotos.Value.newBuilder(); if (value instanceof Double) { builder.setDoubleValue((Double) value); } else if (value instanceof Float) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java index a1ae44f7ef..d289ab1254 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java @@ -38,7 +38,7 @@ import com.apple.foundationdb.record.metadata.expressions.InvertibleFunctionKeyExpression; import com.apple.foundationdb.record.metadata.expressions.TupleFieldsHelper; import com.apple.foundationdb.record.planprotos.PComparison; -import com.apple.foundationdb.record.planprotos.PComparison.PComparisonType; +import com.apple.foundationdb.record.planprotos.PComparisonType; import com.apple.foundationdb.record.planprotos.PInvertedFunctionComparison; import com.apple.foundationdb.record.planprotos.PListComparison; import com.apple.foundationdb.record.planprotos.PMultiColumnComparison; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java index c5a8ddee35..c7cd10c1f1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java @@ -40,7 +40,7 @@ import com.apple.foundationdb.record.planprotos.PIndexKeyValueToPartialRecord.PFieldCopier; import com.apple.foundationdb.record.planprotos.PIndexKeyValueToPartialRecord.PFieldWithValueCopier; import com.apple.foundationdb.record.planprotos.PIndexKeyValueToPartialRecord.PMessageCopier; -import com.apple.foundationdb.record.planprotos.PIndexKeyValueToPartialRecord.PTupleSource; +import com.apple.foundationdb.record.planprotos.PTupleSource; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.predicates.ConstantPredicate; diff --git a/fdb-record-layer-core/src/main/proto/planner_debugger.proto b/fdb-record-layer-core/src/main/proto/planner_debugger.proto index 72864878c3..73019cd560 100644 --- a/fdb-record-layer-core/src/main/proto/planner_debugger.proto +++ b/fdb-record-layer-core/src/main/proto/planner_debugger.proto @@ -21,9 +21,6 @@ syntax = "proto2"; package com.apple.foundationdb.record.query.plan.cascades.debug.eventprotos; import "google/protobuf/descriptor.proto"; -import "google/protobuf/any.proto"; -import "record_metadata.proto"; -import "record_query_plan.proto"; option java_outer_classname = "PlannerDebuggerProto"; option java_multiple_files = true; diff --git a/fdb-record-layer-core/src/main/proto/record_metadata.proto b/fdb-record-layer-core/src/main/proto/record_metadata.proto index df69e6dca2..7d04612b65 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata.proto @@ -21,6 +21,7 @@ syntax = "proto2"; package com.apple.foundationdb.record; import "google/protobuf/descriptor.proto"; +import "record_value.proto"; option java_outer_classname = "RecordMetaDataProto"; @@ -71,7 +72,7 @@ message RecordType { required string name = 1; optional KeyExpression primary_key = 2; optional int32 since_version = 3; - optional Value explicit_key = 4; + optional com.apple.foundationdb.record.planprotos.Value explicit_key = 4; extensions 1000 to 2000; } @@ -102,7 +103,8 @@ message MetaData { } message UDF { - optional Function function = 1; + optional string name = 1; + optional com.apple.foundationdb.record.planprotos.PValue value = 2; } message Then { @@ -164,17 +166,6 @@ message Version { message RecordTypeKey { } -message Value { - // TODO: This should be a oneof {} - optional double double_value = 1; - optional float float_value = 2; - optional int64 long_value = 3; - optional bool bool_value = 4; - optional string string_value = 5; - optional bytes bytes_value = 6; - optional int32 int_value = 7; -} - message Function { required string name = 1; required KeyExpression arguments = 2; @@ -189,7 +180,7 @@ message KeyExpression { optional Empty empty = 5; optional Split split = 6; optional Version version = 7; - optional Value value = 8; + optional com.apple.foundationdb.record.planprotos.Value value = 8; optional Function function = 9; optional KeyWithValue key_with_value = 10; optional RecordTypeKey record_type_key = 11; @@ -199,7 +190,7 @@ message KeyExpression { message JoinedRecordType { optional string name = 1; - optional Value record_type_key = 4; + optional com.apple.foundationdb.record.planprotos.Value record_type_key = 4; message JoinConstituent { optional string name = 1; @@ -220,7 +211,7 @@ message JoinedRecordType { message UnnestedRecordType { optional string name = 1; - optional Value record_type_key = 2; + optional com.apple.foundationdb.record.planprotos.Value record_type_key = 2; message NestedConstituent { optional string name = 1; @@ -248,7 +239,7 @@ enum ComparisonType { message SimpleComparison { optional ComparisonType type = 1; - optional Value operand = 2; + optional com.apple.foundationdb.record.planprotos.Value operand = 2; } // in Comparisons.Comparison we have a special NullComparison that takes care of comparing diff --git a/fdb-record-layer-core/src/main/proto/record_metadata_options.proto b/fdb-record-layer-core/src/main/proto/record_metadata_options.proto index 000d13efd1..0a8e108786 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata_options.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata_options.proto @@ -24,6 +24,7 @@ option java_outer_classname = "RecordMetaDataOptionsProto"; import "google/protobuf/descriptor.proto"; import "record_metadata.proto"; +import "record_value.proto"; message SchemaOptions { optional bool split_long_records = 3; @@ -43,7 +44,7 @@ message RecordTypeOptions { } optional Usage usage = 1 [default = UNSET]; optional int32 since_version = 2; // This record_type is introduced on this metadata version - optional Value record_type_key = 3; + optional com.apple.foundationdb.record.planprotos.Value record_type_key = 3; } extend google.protobuf.MessageOptions { diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 54fd40c410..3e1725e08e 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -22,928 +22,17 @@ syntax = "proto2"; package com.apple.foundationdb.record.planprotos; import "google/protobuf/descriptor.proto"; import "google/protobuf/any.proto"; -import "record_query_runtime.proto"; import "record_metadata.proto"; +import "record_value.proto"; option java_outer_classname = "RecordQueryPlanProto"; option java_multiple_files = true; -// -// Type system -// -// Note that the subtypes are nested messages and we don't define an extension point because PType should be considered -// a sealed treat that cannot be extended. -// -message PType { - enum PTypeCode { - UNKNOWN = 1; - ANY = 2; - NULL = 3; - BOOLEAN = 4; - BYTES = 5; - DOUBLE = 6; - FLOAT = 7; - INT = 8; - LONG = 9; - STRING = 10; - VERSION = 11; - ENUM = 12; - RECORD = 13; - ARRAY = 14; - RELATION = 15; - NONE = 16; - } - - message PPrimitiveType { - optional PTypeCode type_code = 1; - optional bool is_nullable = 2; - } - - message PNullType { - // nothing - } - - message PNoneType { - // nothing - } - - message PAnyType { - // nothing - } - - message PAnyRecordType { - optional bool is_nullable = 1; - } - - message PEnumType { - message PEnumValue { - optional string name = 1; - optional int32 number = 2; - } - - optional bool is_nullable = 1; - repeated PEnumValue enum_values = 2; - optional string name = 3; // referential name -- not used in the planner - } - - message PRecordType { - message PField { - optional PType field_type = 1; - optional string field_name = 2; - optional int32 field_index = 3; - } - - optional int32 reference_id = 1; - optional string name = 2; // referential name -- not used in the planner - optional bool is_nullable = 3; - repeated PField fields = 4; - } - - message PRelationType { - optional PType inner_type = 1; - } - - message PArrayType { - optional bool is_nullable = 2; - optional PType element_type = 3; - } - - oneof specific_type { - PPrimitiveType primitive_type = 1; - PNullType null_type = 2; - PNoneType none_type = 3; - PAnyType any_type = 4; - PEnumType enum_type = 5; - PRecordType record_type = 6; - PRelationType relation_type = 7; - PArrayType array_type = 8; - PAnyRecordType any_record_type = 9; - } -} - -// -// Coercion Tries -// -message PCoercionTrieNode { - message IntChildPair { - optional int32 index = 1; - optional PCoercionTrieNode child_coercion_trie_node = 2; - } - optional bool children_map_is_null = 1; - repeated IntChildPair child_pair = 2; - optional PCoercionBiFunction value = 3; -} - -message PCoercionBiFunction { - extensions 5000 to max; - oneof specific_function { - google.protobuf.Any additional_coercion_bi_functions = 1; - PPrimitiveCoercionBiFunction primitive_coercion_bi_function = 2; - PArrayCoercionBiFunction array_coercion_bi_function = 3; - } -} - -message PPrimitiveCoercionBiFunction { - enum PPhysicalOperator { - INT_TO_LONG = 1; - INT_TO_FLOAT = 2; - INT_TO_DOUBLE = 3; - LONG_TO_FLOAT = 4; - LONG_TO_DOUBLE = 5; - FLOAT_TO_DOUBLE = 6; - NULL_TO_INT = 7; - NULL_TO_LONG = 8; - NULL_TO_FLOAT = 9; - NULL_TO_DOUBLE = 10; - NULL_TO_BOOLEAN = 11; - NULL_TO_STRING = 12; - NULL_TO_ARRAY = 13; - NULL_TO_RECORD = 14; - NONE_TO_ARRAY = 15; - NULL_TO_ENUM = 16; - STRING_TO_ENUM = 17; - } - optional PPhysicalOperator operator = 1; -} - -message PArrayCoercionBiFunction { - optional PType from_array_type = 1; - optional PType to_array_type = 2; - optional PCoercionTrieNode elements_trie = 3; -} - -message PTransformationTrieNode { - message IntChildPair { - optional int32 index = 1; - optional PTransformationTrieNode child_transformation_trie_node = 2; - } - optional bool children_map_is_null = 1; - repeated IntChildPair child_pair = 2; - optional PValue value = 3; -} - -message PFieldAccessTrieNode { - message PResolvedAccessorChildPair { - optional PFieldPath.PResolvedAccessor resolved_accessor = 1; - optional PFieldAccessTrieNode child_field_access_trie_node = 2; - } - optional bool children_map_is_null = 1; - repeated PResolvedAccessorChildPair child_pair = 2; - optional PType type = 3; -} - -// -// Values -// -message PValue { - extensions 5000 to max; - oneof specific_value { - google.protobuf.Any additional_values = 1; - PLightArrayConstructorValue light_array_constructor_value = 2; - PAndOrValue and_or_value = 3; - PArithmeticValue arithmetic_value = 4; - PConditionSelectorValue condition_selector_value = 5; - PConstantObjectValue constant_object_value = 6; - PConstantValue constant_value = 7; - PCountValue count_value = 8; - PDerivedValue derived_value = 9; - PEmptyValue empty_value = 10; - PExistsValue exists_value = 11; - PFieldValue field_value = 12; - PIndexedValue indexed_value = 13; - PMaxEverValue max_ever_value = 14; - PMinEverValue min_ever_value = 15; - PInOpValue in_op_value = 16; - PLikeOperatorValue like_operator_value = 17; - PLiteralValue literal_value = 18; - PNotValue not_value = 19; - PNullValue null_value = 20; - PNumericAggregationValue.PSum numeric_aggregation_value_sum = 21; - PNumericAggregationValue.PAvg numeric_aggregation_value_avg = 22; - PNumericAggregationValue.PMin numeric_aggregation_value_min = 23; - PNumericAggregationValue.PMax numeric_aggregation_value_max = 24; - PObjectValue object_value = 25; - POfTypeValue of_type_value = 26; - PPatternForLikeValue pattern_for_like_value = 27; - PPickValue pick_value = 28; - PPromoteValue promote_value = 29; - PQuantifiedObjectValue quantified_object_value = 30; - PQueriedValue queried_value = 31; - PRankValue rank_value = 32; - PRecordConstructorValue record_constructor_value = 33; - PRecordTypeValue record_type_value = 34; - PBinaryRelOpValue binary_rel_op_value = 35; - PUnaryRelOpValue unary_rel_op_value = 36; - PVariadicFunctionValue variadic_function_value = 37; - PVersionValue version_value = 38; - PFirstOrDefaultValue first_or_default_value = 39; - PThrowsValue throws_value = 40; - PIndexEntryObjectValue index_entry_object_value = 41; - PToOrderedBytesValue to_ordered_bytes_value = 42; - PFromOrderedBytesValue from_ordered_bytes_value = 43; - PCollateValue collate_value = 44; - PNumericAggregationValue.PBitmapConstructAgg numeric_aggregation_value_bitmap_construct_agg = 45; - } -} - -message PAbstractArrayConstructorValue { - repeated PValue children = 1; - optional PType element_type = 2; -} - -message PLightArrayConstructorValue { - optional PAbstractArrayConstructorValue super = 1; -} - -message PAndOrValue { - enum POperator { - AND = 1; - OR = 2; - } - optional string function_name = 1; - optional PValue left_child = 2; - optional PValue right_child = 3; - optional POperator operator = 4; -} - -message PArithmeticValue { - enum PPhysicalOperator { - ADD_II = 1; - ADD_IL = 2; - ADD_IF = 3; - ADD_ID = 4; - ADD_IS = 5; - ADD_LI = 6; - ADD_LL = 7; - ADD_LF = 8; - ADD_LD = 9; - ADD_LS = 10; - ADD_FI = 11; - ADD_FL = 12; - ADD_FF = 13; - ADD_FD = 14; - ADD_FS = 15; - ADD_DI = 16; - ADD_DL = 17; - ADD_DF = 18; - ADD_DD = 19; - ADD_DS = 20; - ADD_SI = 21; - ADD_SL = 22; - ADD_SF = 23; - ADD_SD = 24; - ADD_SS = 25; - - SUB_II = 26; - SUB_IL = 27; - SUB_IF = 28; - SUB_ID = 29; - SUB_LI = 30; - SUB_LL = 31; - SUB_LF = 32; - SUB_LD = 33; - SUB_FI = 34; - SUB_FL = 35; - SUB_FF = 36; - SUB_FD = 37; - SUB_DI = 38; - SUB_DL = 39; - SUB_DF = 40; - SUB_DD = 41; - - MUL_II = 42; // my favorite - MUL_IL = 43; - MUL_IF = 44; - MUL_ID = 45; - MUL_LI = 46; - MUL_LL = 47; - MUL_LF = 48; - MUL_LD = 49; - MUL_FI = 50; - MUL_FL = 51; - MUL_FF = 52; - MUL_FD = 53; - MUL_DI = 54; - MUL_DL = 55; - MUL_DF = 56; - MUL_DD = 57; - - DIV_II = 58; - DIV_IL = 59; - DIV_IF = 60; - DIV_ID = 61; - DIV_LI = 62; - DIV_LL = 63; - DIV_LF = 64; - DIV_LD = 65; - DIV_FI = 66; - DIV_FL = 67; - DIV_FF = 68; - DIV_FD = 69; - DIV_DI = 70; - DIV_DL = 71; - DIV_DF = 72; - DIV_DD = 73; - - MOD_II = 74; - MOD_IL = 75; - MOD_IF = 76; - MOD_ID = 77; - MOD_LI = 78; - MOD_LL = 79; - MOD_LF = 80; - MOD_LD = 81; - MOD_FI = 82; - MOD_FL = 83; - MOD_FF = 84; - MOD_FD = 85; - MOD_DI = 86; - MOD_DL = 87; - MOD_DF = 88; - MOD_DD = 89; - - BITOR_II = 90; - BITOR_IL = 91; - BITOR_LI = 92; - BITOR_LL = 93; - - BITAND_II = 94; - BITAND_IL = 95; - BITAND_LI = 96; - BITAND_LL = 97; - - BITXOR_II = 98; - BITXOR_IL = 99; - BITXOR_LI = 100; - BITXOR_LL = 101; - - BITMAP_BUCKET_OFFSET_LI = 102; - BITMAP_BUCKET_OFFSET_II = 103; - BITMAP_BUCKET_NUMBER_LI = 104; - BITMAP_BUCKET_NUMBER_II = 105; - BITMAP_BIT_POSITION_LI = 106; - BITMAP_BIT_POSITION_II = 107; - } - - optional PPhysicalOperator operator = 1; - optional PValue left_child = 2; - optional PValue right_child = 3; -} - -message PConditionSelectorValue { - repeated PValue implications = 1; -} - -message PConstantObjectValue { - optional string alias = 1; - optional string constant_id= 2; - optional PType result_type = 3; -} - -message PConstantValue { - optional PValue value = 1; -} - -message PCountValue { - enum PPhysicalOperator { - COUNT = 1; - COUNT_STAR = 2; - } - optional PPhysicalOperator operator = 1; - optional PValue child = 2; -} - -message PDerivedValue { - repeated PValue children = 1; - optional PType result_type = 2; -} - -message PEmptyValue { - // nothing -} - -message PExistsValue { - optional PQuantifiedObjectValue child = 1; // deprecated - optional string alias = 2; -} - -message PFieldValue { - optional PValue child_value = 1; - optional PFieldPath field_path = 2; -} - -message PFieldPath { - message PResolvedAccessor { - optional string name = 1; - optional int32 ordinal = 2; - optional PType type = 3; - } - repeated PResolvedAccessor field_accessors = 1; -} - -message PFirstOrDefaultValue { - optional PValue child_value = 1; - optional PValue on_empty_result_value = 2; -} - -message PFromOrderedBytesValue { - optional PValue child = 1; - optional PDirection direction = 2; - optional PType result_type = 3; -} - -enum PDirection { - ASC_NULLS_FIRST = 1; - ASC_NULLS_LAST = 2; - DESC_NULLS_FIRST = 3; - DESC_NULLS_LAST = 4; -} - -message PIndexedValue { - optional PType result_type = 1; -} - -message PIndexEntryObjectValue { - optional string index_entry_alias = 1; - optional PIndexKeyValueToPartialRecord.PTupleSource source = 2; - repeated int32 ordinal_path = 3; - optional PType result_type = 4; -} - -message PIndexOnlyAggregateValue { // abstract - enum PPhysicalOperator { - MAX_EVER_LONG = 1; - MIN_EVER_LONG = 2; - } - optional PPhysicalOperator operator = 1; - optional PValue child = 2; -} - -message PInOpValue { - optional PValue probe_value = 1; - optional PValue in_array_value = 2; -} - -message PMaxEverValue { - optional PIndexOnlyAggregateValue super = 1; -} - -message PMinEverValue { - optional PIndexOnlyAggregateValue super = 1; -} - -message PLikeOperatorValue { - optional PValue src_child = 1; - optional PValue pattern_child = 2; -} - -message PLiteralValue { - optional PType result_type = 1; - optional PComparableObject value = 2; -} - -message PNotValue { - optional PValue child = 1; -} - -message PNullValue { - optional PType result_type = 1; -} - -message PNumericAggregationValue { - enum PPhysicalOperator { - SUM_I = 1; - SUM_L = 2; - SUM_F = 3; - SUM_D = 4; - - AVG_I = 5; - AVG_L = 6; - AVG_F = 7; - AVG_D = 8; - - MIN_I = 9; - MIN_L = 10; - MIN_F = 11; - MIN_D = 12; - - MAX_I = 13; - MAX_L = 14; - MAX_F = 15; - MAX_D = 16; - - BITMAP_CONSTRUCT_AGG_L = 17; - BITMAP_CONSTRUCT_AGG_I = 18; - } - optional PPhysicalOperator operator = 1; - optional PValue child = 2; - - message PBitmapConstructAgg { - optional PNumericAggregationValue super = 1; - } - message PSum { - optional PNumericAggregationValue super = 1; - } - message PAvg { - optional PNumericAggregationValue super = 1; - } - message PMin { - optional PNumericAggregationValue super = 1; - } - message PMax { - optional PNumericAggregationValue super = 1; - } -} - -message PObjectValue { - optional string alias = 1; - optional PType result_type = 2; -} - -message POfTypeValue { - optional PValue child = 1; - optional PType expected_type = 2; -} - -message PPatternForLikeValue { - optional PValue pattern_child = 1; - optional PValue escape_child = 2; -} - -message PPickValue { - optional PValue selector_value = 1; - repeated PValue alternative_values = 2; - optional PType result_type = 3; -} - -message PPromoteValue { - optional PValue in_value = 1; - optional PType promote_to_type = 2; - optional PCoercionTrieNode promotion_trie = 3; -} - -message PQuantifiedObjectValue { - optional string alias = 1; - optional PType result_type = 2; -} - -message PQueriedValue { - optional PType result_type = 1; - optional bool has_record_type_names = 3; - repeated string record_type_names = 2; -} - -message PRankValue { - optional PWindowedValue super = 1; -} - -message PRecordConstructorValue { - message PColumn { - optional PType.PRecordType.PField field = 1; - optional PValue value = 2; - } - optional PType result_type = 1; - repeated PColumn columns = 2; -} - -message PRecordTypeValue { - optional string alias = 1; // deprecated - optional PValue in = 2; -} - -message PRelOpValue { - optional string function_name = 1; - optional PComparison.PComparisonType comparison_type = 2; - repeated PValue children = 3; -} - -message PBinaryRelOpValue { - enum PBinaryPhysicalOperator { - EQ_BU = 1; - EQ_BB = 2; - EQ_IU = 3; - EQ_II = 4; - EQ_IL = 5; - EQ_IF = 6; - EQ_ID = 7; - EQ_LU = 8; - EQ_LI = 9; - EQ_LL = 10; - EQ_LF = 11; - EQ_LD = 12; - EQ_FU = 13; - EQ_FI = 14; - EQ_FL = 15; - EQ_FF = 16; - EQ_FD = 17; - EQ_DU = 18; - EQ_DI = 19; - EQ_DL = 20; - EQ_DF = 21; - EQ_DD = 22; - EQ_SU = 23; - EQ_SS = 24; - EQ_UU = 25; - EQ_UB = 26; - EQ_UI = 27; - EQ_UL = 28; - EQ_UF = 29; - EQ_UD = 30; - EQ_US = 31; - EQ_UV = 32; - EQ_VU = 33; - EQ_VV = 34; - NEQ_BU = 35; - NEQ_BB = 36; - NEQ_IU = 37; - NEQ_II = 38; - NEQ_IL = 39; - NEQ_IF = 40; - NEQ_ID = 41; - NEQ_LU = 42; - NEQ_LI = 43; - NEQ_LL = 44; - NEQ_LF = 45; - NEQ_LD = 46; - NEQ_FU = 47; - NEQ_FI = 48; - NEQ_FL = 49; - NEQ_FF = 50; - NEQ_FD = 51; - NEQ_DU = 52; - NEQ_DI = 53; - NEQ_DL = 54; - NEQ_DF = 55; - NEQ_DD = 56; - NEQ_SU = 57; - NEQ_SS = 58; - NEQ_UU = 59; - NEQ_UB = 60; - NEQ_UI = 61; - NEQ_UL = 62; - NEQ_UF = 63; - NEQ_UD = 64; - NEQ_US = 65; - NEQ_UV = 66; - NEQ_VU = 67; - NEQ_VV = 68; - LT_IU = 69; - LT_II = 70; - LT_IL = 71; - LT_IF = 72; - LT_ID = 73; - LT_LU = 74; - LT_LI = 75; - LT_LL = 76; - LT_LF = 77; - LT_LD = 78; - LT_FU = 79; - LT_FI = 80; - LT_FL = 81; - LT_FF = 82; - LT_FD = 83; - LT_DU = 84; - LT_DI = 85; - LT_DL = 86; - LT_DF = 87; - LT_DD = 88; - LT_SU = 89; - LT_SS = 90; - LT_UU = 91; - LT_UB = 92; - LT_UI = 93; - LT_UL = 94; - LT_UF = 95; - LT_UD = 96; - LT_US = 97; - LT_UV = 98; - LT_VU = 99; - LT_VV = 100; - LTE_IU = 101; - LTE_II = 102; - LTE_IL = 103; - LTE_IF = 104; - LTE_ID = 105; - LTE_LU = 106; - LTE_LI = 107; - LTE_LL = 108; - LTE_LF = 109; - LTE_LD = 110; - LTE_FU = 111; - LTE_FI = 112; - LTE_FL = 113; - LTE_FF = 114; - LTE_FD = 115; - LTE_DU = 116; - LTE_DI = 117; - LTE_DL = 118; - LTE_DF = 119; - LTE_DD = 120; - LTE_SU = 121; - LTE_SS = 122; - LTE_UU = 123; - LTE_UB = 124; - LTE_UI = 125; - LTE_UL = 126; - LTE_UF = 127; - LTE_UD = 128; - LTE_US = 129; - LTE_UV = 130; - LTE_VU = 131; - LTE_VV = 132; - GT_IU = 133; - GT_II = 134; - GT_IL = 135; - GT_IF = 136; - GT_ID = 137; - GT_LU = 138; - GT_LI = 139; - GT_LL = 140; - GT_LF = 141; - GT_LD = 142; - GT_FU = 143; - GT_FI = 144; - GT_FL = 145; - GT_FF = 146; - GT_FD = 147; - GT_DU = 148; - GT_DI = 149; - GT_DL = 150; - GT_DF = 151; - GT_DD = 152; - GT_SU = 153; - GT_SS = 154; - GT_UU = 155; - GT_UB = 156; - GT_UI = 157; - GT_UL = 158; - GT_UF = 159; - GT_UD = 160; - GT_US = 161; - GT_UV = 162; - GT_VU = 163; - GT_VV = 164; - GTE_IU = 165; - GTE_II = 166; - GTE_IL = 167; - GTE_IF = 168; - GTE_ID = 169; - GTE_LU = 170; - GTE_LI = 171; - GTE_LL = 172; - GTE_LF = 173; - GTE_LD = 174; - GTE_FU = 175; - GTE_FI = 176; - GTE_FL = 177; - GTE_FF = 178; - GTE_FD = 179; - GTE_DU = 180; - GTE_DI = 181; - GTE_DL = 182; - GTE_DF = 183; - GTE_DD = 184; - GTE_SU = 185; - GTE_SS = 186; - GTE_UU = 187; - GTE_UB = 188; - GTE_UI = 189; - GTE_UL = 190; - GTE_UF = 191; - GTE_UD = 192; - GTE_US = 193; - GTE_UV = 194; - GTE_VU = 195; - GTE_VV = 196; - - EQ_BYU = 197; - EQ_BYBY = 198; - EQ_UBY = 199; - NEQ_BYU = 200; - NEQ_BYBY = 201; - NEQ_UBY = 202; - LT_BYU = 203; - LT_BYBY = 204; - LTE_BYU = 205; - LTE_BYBY = 206; - GT_BYU = 207; - GT_BYBY = 208; - GTE_BYU = 209; - GTE_BYBY = 210; - - } - optional PRelOpValue super = 1; - optional PBinaryPhysicalOperator operator = 2; -} - -message PThrowsValue { - optional PType result_type = 1; -} - -message PToOrderedBytesValue { - optional PValue child = 1; - optional PDirection direction = 2; -} - -message PUnaryRelOpValue { - enum PUnaryPhysicalOperator { - IS_NULL_UI = 1; - IS_NULL_II = 2; - IS_NULL_LI = 3; - IS_NULL_FI = 4; - IS_NULL_DI = 5; - IS_NULL_SS = 6; - IS_NULL_BI = 7; - - IS_NOT_NULL_UI = 8; - IS_NOT_NULL_II = 9; - IS_NOT_NULL_LI = 10; - IS_NOT_NULL_FI = 11; - IS_NOT_NULL_DI = 12; - IS_NOT_NULL_SS = 13; - IS_NOT_NULL_BI = 14; - - IS_NULL_BY = 15; - IS_NOT_NULL_BY = 16; - } - optional PRelOpValue super = 1; - optional PUnaryPhysicalOperator operator = 2; -} - -message PUdfValue { // abstract - repeated PValue children = 1; - optional PType result_type = 2; -} - -message PVariadicFunctionValue { - enum PPhysicalOperator { - GREATEST_INT = 1; - GREATEST_LONG = 2; - GREATEST_BOOLEAN = 3; - GREATEST_STRING = 4; - GREATEST_FLOAT = 5; - GREATEST_DOUBLE = 6; - LEAST_INT = 7; - LEAST_LONG = 8; - LEAST_BOOLEAN = 9; - LEAST_STRING = 10; - LEAST_FLOAT = 11; - LEAST_DOUBLE = 12; - COALESCE_INT = 13; - COALESCE_LONG = 14; - COALESCE_BOOLEAN = 15; - COALESCE_STRING = 16; - COALESCE_FLOAT = 17; - COALESCE_DOUBLE = 18; - COALESCE_RECORD = 19; - COALESCE_ARRAY = 20; - } - optional PPhysicalOperator operator = 1; - repeated PValue children = 2; -} - -message PVersionValue { - optional string base_alias = 1; -} - -message PWindowedValue { - repeated PValue partitioning_values = 1; - repeated PValue argument_values = 2; -} - -message PCollateValue { - optional string collator_registry = 1; - optional PValue string_child = 2; - optional PValue locale_child = 3; - optional PValue strength_child = 4; -} // // Comparisons // message PComparison { - enum PComparisonType { - EQUALS = 1; - NOT_EQUALS = 2; - LESS_THAN = 3; - LESS_THAN_OR_EQUALS = 4; - GREATER_THAN = 5; - GREATER_THAN_OR_EQUALS = 6; - STARTS_WITH = 7; - NOT_NULL = 8; - IS_NULL = 9; - IN = 10; - TEXT_CONTAINS_ALL = 11; - TEXT_CONTAINS_ALL_WITHIN = 12; - TEXT_CONTAINS_ANY = 13; - TEXT_CONTAINS_PHRASE = 14; - TEXT_CONTAINS_PREFIX = 15; - TEXT_CONTAINS_ALL_PREFIXES = 16; - TEXT_CONTAINS_ANY_PREFIX = 17; - SORT = 18; - LIKE = 19; - } - extensions 5000 to max; oneof specific_comparison { google.protobuf.Any additional_comparisons = 1; @@ -962,12 +51,12 @@ message PComparison { } message PSimpleComparison { - optional PComparison.PComparisonType type = 1; + optional PComparisonType type = 1; optional PComparableObject object = 2; } message PConversionSimpleComparison { - optional PComparison.PComparisonType type = 1; + optional PComparisonType type = 1; optional PComparableObject object = 2; optional KeyExpression conversion = 3; } @@ -980,30 +69,30 @@ message PParameterComparison { CONSTANT = 4; } - optional PComparison.PComparisonType type = 1; + optional PComparisonType type = 1; optional string parameter = 2; optional PBindingKind internal = 3; } message PConversionParameterComparison { - optional PComparison.PComparisonType type = 1; + optional PComparisonType type = 1; optional string parameter = 2; optional PParameterComparison.PBindingKind internal = 3; optional KeyExpression conversion = 4; } message PValueComparison { - optional PComparison.PComparisonType type = 1; + optional PComparisonType type = 1; optional PValue comparandValue = 2; } message PListComparison { - optional PComparison.PComparisonType type = 1; + optional PComparisonType type = 1; repeated PComparableObject comparand = 2; } message PNullComparison { - optional PComparison.PComparisonType type = 1; + optional PComparisonType type = 1; } message POpaqueEqualityComparison { @@ -1017,7 +106,7 @@ message PMultiColumnComparison { message PInvertedFunctionComparison { optional Function function = 1; optional PComparison original_comparison = 2; - optional PComparison.PComparisonType type = 3; + optional PComparisonType type = 3; } message PRecordTypeComparison { @@ -1231,12 +320,6 @@ message PIndexKeyValueToPartialRecord { optional string invertible_function = 5; } - enum PTupleSource { - KEY = 1; - VALUE = 2; - OTHER = 3; - } - message PCopyIfPredicate { message PTruePredicate { // nothing diff --git a/fdb-record-layer-core/src/main/proto/record_query_runtime.proto b/fdb-record-layer-core/src/main/proto/record_query_runtime.proto index 8f4000c88c..5d1cc7acee 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_runtime.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_runtime.proto @@ -21,35 +21,11 @@ syntax = "proto2"; package com.apple.foundationdb.record.planprotos; import "google/protobuf/descriptor.proto"; -import "record_metadata.proto"; +import "record_value.proto"; option java_outer_classname = "RecordQueryRuntimeProto"; option java_multiple_files = true; -message PEnumLightValue { - optional string name = 1; - optional int32 number = 2; -} - -message PUUID { - optional uint64 mostSigBits = 1; - optional uint64 leastSigBits = 2; -} - -message PFDBRecordVersion { - optional bytes raw_bytes = 1; -} - -message PComparableObject { - oneof specific_object { - Value primitive_object = 1; - PEnumLightValue enum_object = 2; - PUUID uuid = 3; - PFDBRecordVersion fdb_record_version = 4; - bytes bytes_as_byte_string = 5; - } -} - message PQueryResult { oneof datum { PComparableObject primitive = 1; diff --git a/fdb-record-layer-core/src/main/proto/record_value.proto b/fdb-record-layer-core/src/main/proto/record_value.proto new file mode 100644 index 0000000000..d2f43b1810 --- /dev/null +++ b/fdb-record-layer-core/src/main/proto/record_value.proto @@ -0,0 +1,962 @@ +syntax = "proto2"; + +package com.apple.foundationdb.record.planprotos; +import "google/protobuf/descriptor.proto"; +import "google/protobuf/any.proto"; + +option java_outer_classname = "RecordMetaDataValueProto"; +option java_multiple_files = true; + +// +// Type system +// +// Note that the subtypes are nested messages and we don't define an extension point because PType should be considered +// a sealed treat that cannot be extended. +// +message PType { + enum PTypeCode { + UNKNOWN = 1; + ANY = 2; + NULL = 3; + BOOLEAN = 4; + BYTES = 5; + DOUBLE = 6; + FLOAT = 7; + INT = 8; + LONG = 9; + STRING = 10; + VERSION = 11; + ENUM = 12; + RECORD = 13; + ARRAY = 14; + RELATION = 15; + NONE = 16; + } + + message PPrimitiveType { + optional PTypeCode type_code = 1; + optional bool is_nullable = 2; + } + + message PNullType { + // nothing + } + + message PNoneType { + // nothing + } + + message PAnyType { + // nothing + } + + message PAnyRecordType { + optional bool is_nullable = 1; + } + + message PEnumType { + message PEnumValue { + optional string name = 1; + optional int32 number = 2; + } + + optional bool is_nullable = 1; + repeated PEnumValue enum_values = 2; + optional string name = 3; // referential name -- not used in the planner + } + + message PRecordType { + message PField { + optional PType field_type = 1; + optional string field_name = 2; + optional int32 field_index = 3; + } + + optional int32 reference_id = 1; + optional string name = 2; // referential name -- not used in the planner + optional bool is_nullable = 3; + repeated PField fields = 4; + } + + message PRelationType { + optional PType inner_type = 1; + } + + message PArrayType { + optional bool is_nullable = 2; + optional PType element_type = 3; + } + + oneof specific_type { + PPrimitiveType primitive_type = 1; + PNullType null_type = 2; + PNoneType none_type = 3; + PAnyType any_type = 4; + PEnumType enum_type = 5; + PRecordType record_type = 6; + PRelationType relation_type = 7; + PArrayType array_type = 8; + PAnyRecordType any_record_type = 9; + } +} + +// +// Coercion Tries +// +message PCoercionTrieNode { + message IntChildPair { + optional int32 index = 1; + optional PCoercionTrieNode child_coercion_trie_node = 2; + } + optional bool children_map_is_null = 1; + repeated IntChildPair child_pair = 2; + optional PCoercionBiFunction value = 3; +} + +message PCoercionBiFunction { + extensions 5000 to max; + oneof specific_function { + google.protobuf.Any additional_coercion_bi_functions = 1; + PPrimitiveCoercionBiFunction primitive_coercion_bi_function = 2; + PArrayCoercionBiFunction array_coercion_bi_function = 3; + } +} + +message PPrimitiveCoercionBiFunction { + enum PPhysicalOperator { + INT_TO_LONG = 1; + INT_TO_FLOAT = 2; + INT_TO_DOUBLE = 3; + LONG_TO_FLOAT = 4; + LONG_TO_DOUBLE = 5; + FLOAT_TO_DOUBLE = 6; + NULL_TO_INT = 7; + NULL_TO_LONG = 8; + NULL_TO_FLOAT = 9; + NULL_TO_DOUBLE = 10; + NULL_TO_BOOLEAN = 11; + NULL_TO_STRING = 12; + NULL_TO_ARRAY = 13; + NULL_TO_RECORD = 14; + NONE_TO_ARRAY = 15; + NULL_TO_ENUM = 16; + STRING_TO_ENUM = 17; + } + optional PPhysicalOperator operator = 1; +} + +message PArrayCoercionBiFunction { + optional PType from_array_type = 1; + optional PType to_array_type = 2; + optional PCoercionTrieNode elements_trie = 3; +} + +message PTransformationTrieNode { + message IntChildPair { + optional int32 index = 1; + optional PTransformationTrieNode child_transformation_trie_node = 2; + } + optional bool children_map_is_null = 1; + repeated IntChildPair child_pair = 2; + optional PValue value = 3; +} + +message PFieldAccessTrieNode { + message PResolvedAccessorChildPair { + optional PFieldPath.PResolvedAccessor resolved_accessor = 1; + optional PFieldAccessTrieNode child_field_access_trie_node = 2; + } + optional bool children_map_is_null = 1; + repeated PResolvedAccessorChildPair child_pair = 2; + optional PType type = 3; +} + + +// +// Values +// +message PValue { + extensions 5000 to max; + oneof specific_value { + google.protobuf.Any additional_values = 1; + PLightArrayConstructorValue light_array_constructor_value = 2; + PAndOrValue and_or_value = 3; + PArithmeticValue arithmetic_value = 4; + PConditionSelectorValue condition_selector_value = 5; + PConstantObjectValue constant_object_value = 6; + PConstantValue constant_value = 7; + PCountValue count_value = 8; + PDerivedValue derived_value = 9; + PEmptyValue empty_value = 10; + PExistsValue exists_value = 11; + PFieldValue field_value = 12; + PIndexedValue indexed_value = 13; + PMaxEverValue max_ever_value = 14; + PMinEverValue min_ever_value = 15; + PInOpValue in_op_value = 16; + PLikeOperatorValue like_operator_value = 17; + PLiteralValue literal_value = 18; + PNotValue not_value = 19; + PNullValue null_value = 20; + PNumericAggregationValue.PSum numeric_aggregation_value_sum = 21; + PNumericAggregationValue.PAvg numeric_aggregation_value_avg = 22; + PNumericAggregationValue.PMin numeric_aggregation_value_min = 23; + PNumericAggregationValue.PMax numeric_aggregation_value_max = 24; + PObjectValue object_value = 25; + POfTypeValue of_type_value = 26; + PPatternForLikeValue pattern_for_like_value = 27; + PPickValue pick_value = 28; + PPromoteValue promote_value = 29; + PQuantifiedObjectValue quantified_object_value = 30; + PQueriedValue queried_value = 31; + PRankValue rank_value = 32; + PRecordConstructorValue record_constructor_value = 33; + PRecordTypeValue record_type_value = 34; + PBinaryRelOpValue binary_rel_op_value = 35; + PUnaryRelOpValue unary_rel_op_value = 36; + PVariadicFunctionValue variadic_function_value = 37; + PVersionValue version_value = 38; + PFirstOrDefaultValue first_or_default_value = 39; + PThrowsValue throws_value = 40; + PIndexEntryObjectValue index_entry_object_value = 41; + PToOrderedBytesValue to_ordered_bytes_value = 42; + PFromOrderedBytesValue from_ordered_bytes_value = 43; + PCollateValue collate_value = 44; + PNumericAggregationValue.PBitmapConstructAgg numeric_aggregation_value_bitmap_construct_agg = 45; + } +} + +message PAbstractArrayConstructorValue { + repeated PValue children = 1; + optional PType element_type = 2; +} + +message PLightArrayConstructorValue { + optional PAbstractArrayConstructorValue super = 1; +} + +message PAndOrValue { + enum POperator { + AND = 1; + OR = 2; + } + optional string function_name = 1; + optional PValue left_child = 2; + optional PValue right_child = 3; + optional POperator operator = 4; +} + +message PArithmeticValue { + enum PPhysicalOperator { + ADD_II = 1; + ADD_IL = 2; + ADD_IF = 3; + ADD_ID = 4; + ADD_IS = 5; + ADD_LI = 6; + ADD_LL = 7; + ADD_LF = 8; + ADD_LD = 9; + ADD_LS = 10; + ADD_FI = 11; + ADD_FL = 12; + ADD_FF = 13; + ADD_FD = 14; + ADD_FS = 15; + ADD_DI = 16; + ADD_DL = 17; + ADD_DF = 18; + ADD_DD = 19; + ADD_DS = 20; + ADD_SI = 21; + ADD_SL = 22; + ADD_SF = 23; + ADD_SD = 24; + ADD_SS = 25; + + SUB_II = 26; + SUB_IL = 27; + SUB_IF = 28; + SUB_ID = 29; + SUB_LI = 30; + SUB_LL = 31; + SUB_LF = 32; + SUB_LD = 33; + SUB_FI = 34; + SUB_FL = 35; + SUB_FF = 36; + SUB_FD = 37; + SUB_DI = 38; + SUB_DL = 39; + SUB_DF = 40; + SUB_DD = 41; + + MUL_II = 42; // my favorite + MUL_IL = 43; + MUL_IF = 44; + MUL_ID = 45; + MUL_LI = 46; + MUL_LL = 47; + MUL_LF = 48; + MUL_LD = 49; + MUL_FI = 50; + MUL_FL = 51; + MUL_FF = 52; + MUL_FD = 53; + MUL_DI = 54; + MUL_DL = 55; + MUL_DF = 56; + MUL_DD = 57; + + DIV_II = 58; + DIV_IL = 59; + DIV_IF = 60; + DIV_ID = 61; + DIV_LI = 62; + DIV_LL = 63; + DIV_LF = 64; + DIV_LD = 65; + DIV_FI = 66; + DIV_FL = 67; + DIV_FF = 68; + DIV_FD = 69; + DIV_DI = 70; + DIV_DL = 71; + DIV_DF = 72; + DIV_DD = 73; + + MOD_II = 74; + MOD_IL = 75; + MOD_IF = 76; + MOD_ID = 77; + MOD_LI = 78; + MOD_LL = 79; + MOD_LF = 80; + MOD_LD = 81; + MOD_FI = 82; + MOD_FL = 83; + MOD_FF = 84; + MOD_FD = 85; + MOD_DI = 86; + MOD_DL = 87; + MOD_DF = 88; + MOD_DD = 89; + + BITOR_II = 90; + BITOR_IL = 91; + BITOR_LI = 92; + BITOR_LL = 93; + + BITAND_II = 94; + BITAND_IL = 95; + BITAND_LI = 96; + BITAND_LL = 97; + + BITXOR_II = 98; + BITXOR_IL = 99; + BITXOR_LI = 100; + BITXOR_LL = 101; + + BITMAP_BUCKET_OFFSET_LI = 102; + BITMAP_BUCKET_OFFSET_II = 103; + BITMAP_BUCKET_NUMBER_LI = 104; + BITMAP_BUCKET_NUMBER_II = 105; + BITMAP_BIT_POSITION_LI = 106; + BITMAP_BIT_POSITION_II = 107; + } + + optional PPhysicalOperator operator = 1; + optional PValue left_child = 2; + optional PValue right_child = 3; +} + +message PConditionSelectorValue { + repeated PValue implications = 1; +} + +message PConstantObjectValue { + optional string alias = 1; + optional string constant_id= 2; + optional PType result_type = 3; +} + +message PConstantValue { + optional PValue value = 1; +} + +message PCountValue { + enum PPhysicalOperator { + COUNT = 1; + COUNT_STAR = 2; + } + optional PPhysicalOperator operator = 1; + optional PValue child = 2; +} + +message PDerivedValue { + repeated PValue children = 1; + optional PType result_type = 2; +} + +message PEmptyValue { + // nothing +} + +message PExistsValue { + optional PQuantifiedObjectValue child = 1; // deprecated + optional string alias = 2; +} + +message PFieldValue { + optional PValue child_value = 1; + optional PFieldPath field_path = 2; +} + +message PFieldPath { + message PResolvedAccessor { + optional string name = 1; + optional int32 ordinal = 2; + optional PType type = 3; + } + repeated PResolvedAccessor field_accessors = 1; +} + +message PFirstOrDefaultValue { + optional PValue child_value = 1; + optional PValue on_empty_result_value = 2; +} + +message PFromOrderedBytesValue { + optional PValue child = 1; + optional PDirection direction = 2; + optional PType result_type = 3; +} + +enum PDirection { + ASC_NULLS_FIRST = 1; + ASC_NULLS_LAST = 2; + DESC_NULLS_FIRST = 3; + DESC_NULLS_LAST = 4; +} + +message PIndexedValue { + optional PType result_type = 1; +} + +message PIndexEntryObjectValue { + optional string index_entry_alias = 1; + optional PTupleSource source = 2; + repeated int32 ordinal_path = 3; + optional PType result_type = 4; +} + +message PIndexOnlyAggregateValue { // abstract + enum PPhysicalOperator { + MAX_EVER_LONG = 1; + MIN_EVER_LONG = 2; + } + optional PPhysicalOperator operator = 1; + optional PValue child = 2; +} + +message PInOpValue { + optional PValue probe_value = 1; + optional PValue in_array_value = 2; +} + +message PMaxEverValue { + optional PIndexOnlyAggregateValue super = 1; +} + +message PMinEverValue { + optional PIndexOnlyAggregateValue super = 1; +} + +message PLikeOperatorValue { + optional PValue src_child = 1; + optional PValue pattern_child = 2; +} + +message PLiteralValue { + optional PType result_type = 1; + optional PComparableObject value = 2; +} + +message PNotValue { + optional PValue child = 1; +} + +message PNullValue { + optional PType result_type = 1; +} + +message PNumericAggregationValue { + enum PPhysicalOperator { + SUM_I = 1; + SUM_L = 2; + SUM_F = 3; + SUM_D = 4; + + AVG_I = 5; + AVG_L = 6; + AVG_F = 7; + AVG_D = 8; + + MIN_I = 9; + MIN_L = 10; + MIN_F = 11; + MIN_D = 12; + + MAX_I = 13; + MAX_L = 14; + MAX_F = 15; + MAX_D = 16; + + BITMAP_CONSTRUCT_AGG_L = 17; + BITMAP_CONSTRUCT_AGG_I = 18; + } + optional PPhysicalOperator operator = 1; + optional PValue child = 2; + + message PBitmapConstructAgg { + optional PNumericAggregationValue super = 1; + } + message PSum { + optional PNumericAggregationValue super = 1; + } + message PAvg { + optional PNumericAggregationValue super = 1; + } + message PMin { + optional PNumericAggregationValue super = 1; + } + message PMax { + optional PNumericAggregationValue super = 1; + } +} + +message PObjectValue { + optional string alias = 1; + optional PType result_type = 2; +} + +message POfTypeValue { + optional PValue child = 1; + optional PType expected_type = 2; +} + +message PPatternForLikeValue { + optional PValue pattern_child = 1; + optional PValue escape_child = 2; +} + +message PPickValue { + optional PValue selector_value = 1; + repeated PValue alternative_values = 2; + optional PType result_type = 3; +} + +message PPromoteValue { + optional PValue in_value = 1; + optional PType promote_to_type = 2; + optional PCoercionTrieNode promotion_trie = 3; +} + +message PQuantifiedObjectValue { + optional string alias = 1; + optional PType result_type = 2; +} + +message PQueriedValue { + optional PType result_type = 1; + optional bool has_record_type_names = 3; + repeated string record_type_names = 2; +} + +message PRankValue { + optional PWindowedValue super = 1; +} + +message PRecordConstructorValue { + message PColumn { + optional PType.PRecordType.PField field = 1; + optional PValue value = 2; + } + optional PType result_type = 1; + repeated PColumn columns = 2; +} + +message PRecordTypeValue { + optional string alias = 1; // deprecated + optional PValue in = 2; +} + +message PRelOpValue { + optional string function_name = 1; + optional PComparisonType comparison_type = 2; + repeated PValue children = 3; +} + +message PBinaryRelOpValue { + enum PBinaryPhysicalOperator { + EQ_BU = 1; + EQ_BB = 2; + EQ_IU = 3; + EQ_II = 4; + EQ_IL = 5; + EQ_IF = 6; + EQ_ID = 7; + EQ_LU = 8; + EQ_LI = 9; + EQ_LL = 10; + EQ_LF = 11; + EQ_LD = 12; + EQ_FU = 13; + EQ_FI = 14; + EQ_FL = 15; + EQ_FF = 16; + EQ_FD = 17; + EQ_DU = 18; + EQ_DI = 19; + EQ_DL = 20; + EQ_DF = 21; + EQ_DD = 22; + EQ_SU = 23; + EQ_SS = 24; + EQ_UU = 25; + EQ_UB = 26; + EQ_UI = 27; + EQ_UL = 28; + EQ_UF = 29; + EQ_UD = 30; + EQ_US = 31; + EQ_UV = 32; + EQ_VU = 33; + EQ_VV = 34; + NEQ_BU = 35; + NEQ_BB = 36; + NEQ_IU = 37; + NEQ_II = 38; + NEQ_IL = 39; + NEQ_IF = 40; + NEQ_ID = 41; + NEQ_LU = 42; + NEQ_LI = 43; + NEQ_LL = 44; + NEQ_LF = 45; + NEQ_LD = 46; + NEQ_FU = 47; + NEQ_FI = 48; + NEQ_FL = 49; + NEQ_FF = 50; + NEQ_FD = 51; + NEQ_DU = 52; + NEQ_DI = 53; + NEQ_DL = 54; + NEQ_DF = 55; + NEQ_DD = 56; + NEQ_SU = 57; + NEQ_SS = 58; + NEQ_UU = 59; + NEQ_UB = 60; + NEQ_UI = 61; + NEQ_UL = 62; + NEQ_UF = 63; + NEQ_UD = 64; + NEQ_US = 65; + NEQ_UV = 66; + NEQ_VU = 67; + NEQ_VV = 68; + LT_IU = 69; + LT_II = 70; + LT_IL = 71; + LT_IF = 72; + LT_ID = 73; + LT_LU = 74; + LT_LI = 75; + LT_LL = 76; + LT_LF = 77; + LT_LD = 78; + LT_FU = 79; + LT_FI = 80; + LT_FL = 81; + LT_FF = 82; + LT_FD = 83; + LT_DU = 84; + LT_DI = 85; + LT_DL = 86; + LT_DF = 87; + LT_DD = 88; + LT_SU = 89; + LT_SS = 90; + LT_UU = 91; + LT_UB = 92; + LT_UI = 93; + LT_UL = 94; + LT_UF = 95; + LT_UD = 96; + LT_US = 97; + LT_UV = 98; + LT_VU = 99; + LT_VV = 100; + LTE_IU = 101; + LTE_II = 102; + LTE_IL = 103; + LTE_IF = 104; + LTE_ID = 105; + LTE_LU = 106; + LTE_LI = 107; + LTE_LL = 108; + LTE_LF = 109; + LTE_LD = 110; + LTE_FU = 111; + LTE_FI = 112; + LTE_FL = 113; + LTE_FF = 114; + LTE_FD = 115; + LTE_DU = 116; + LTE_DI = 117; + LTE_DL = 118; + LTE_DF = 119; + LTE_DD = 120; + LTE_SU = 121; + LTE_SS = 122; + LTE_UU = 123; + LTE_UB = 124; + LTE_UI = 125; + LTE_UL = 126; + LTE_UF = 127; + LTE_UD = 128; + LTE_US = 129; + LTE_UV = 130; + LTE_VU = 131; + LTE_VV = 132; + GT_IU = 133; + GT_II = 134; + GT_IL = 135; + GT_IF = 136; + GT_ID = 137; + GT_LU = 138; + GT_LI = 139; + GT_LL = 140; + GT_LF = 141; + GT_LD = 142; + GT_FU = 143; + GT_FI = 144; + GT_FL = 145; + GT_FF = 146; + GT_FD = 147; + GT_DU = 148; + GT_DI = 149; + GT_DL = 150; + GT_DF = 151; + GT_DD = 152; + GT_SU = 153; + GT_SS = 154; + GT_UU = 155; + GT_UB = 156; + GT_UI = 157; + GT_UL = 158; + GT_UF = 159; + GT_UD = 160; + GT_US = 161; + GT_UV = 162; + GT_VU = 163; + GT_VV = 164; + GTE_IU = 165; + GTE_II = 166; + GTE_IL = 167; + GTE_IF = 168; + GTE_ID = 169; + GTE_LU = 170; + GTE_LI = 171; + GTE_LL = 172; + GTE_LF = 173; + GTE_LD = 174; + GTE_FU = 175; + GTE_FI = 176; + GTE_FL = 177; + GTE_FF = 178; + GTE_FD = 179; + GTE_DU = 180; + GTE_DI = 181; + GTE_DL = 182; + GTE_DF = 183; + GTE_DD = 184; + GTE_SU = 185; + GTE_SS = 186; + GTE_UU = 187; + GTE_UB = 188; + GTE_UI = 189; + GTE_UL = 190; + GTE_UF = 191; + GTE_UD = 192; + GTE_US = 193; + GTE_UV = 194; + GTE_VU = 195; + GTE_VV = 196; + + EQ_BYU = 197; + EQ_BYBY = 198; + EQ_UBY = 199; + NEQ_BYU = 200; + NEQ_BYBY = 201; + NEQ_UBY = 202; + LT_BYU = 203; + LT_BYBY = 204; + LTE_BYU = 205; + LTE_BYBY = 206; + GT_BYU = 207; + GT_BYBY = 208; + GTE_BYU = 209; + GTE_BYBY = 210; + + } + optional PRelOpValue super = 1; + optional PBinaryPhysicalOperator operator = 2; +} + +message PThrowsValue { + optional PType result_type = 1; +} + +message PToOrderedBytesValue { + optional PValue child = 1; + optional PDirection direction = 2; +} + +message PUnaryRelOpValue { + enum PUnaryPhysicalOperator { + IS_NULL_UI = 1; + IS_NULL_II = 2; + IS_NULL_LI = 3; + IS_NULL_FI = 4; + IS_NULL_DI = 5; + IS_NULL_SS = 6; + IS_NULL_BI = 7; + + IS_NOT_NULL_UI = 8; + IS_NOT_NULL_II = 9; + IS_NOT_NULL_LI = 10; + IS_NOT_NULL_FI = 11; + IS_NOT_NULL_DI = 12; + IS_NOT_NULL_SS = 13; + IS_NOT_NULL_BI = 14; + + IS_NULL_BY = 15; + IS_NOT_NULL_BY = 16; + } + optional PRelOpValue super = 1; + optional PUnaryPhysicalOperator operator = 2; +} + +message PUdfValue { // abstract + repeated PValue children = 1; + optional PType result_type = 2; +} + +message PVariadicFunctionValue { + enum PPhysicalOperator { + GREATEST_INT = 1; + GREATEST_LONG = 2; + GREATEST_BOOLEAN = 3; + GREATEST_STRING = 4; + GREATEST_FLOAT = 5; + GREATEST_DOUBLE = 6; + LEAST_INT = 7; + LEAST_LONG = 8; + LEAST_BOOLEAN = 9; + LEAST_STRING = 10; + LEAST_FLOAT = 11; + LEAST_DOUBLE = 12; + COALESCE_INT = 13; + COALESCE_LONG = 14; + COALESCE_BOOLEAN = 15; + COALESCE_STRING = 16; + COALESCE_FLOAT = 17; + COALESCE_DOUBLE = 18; + COALESCE_RECORD = 19; + COALESCE_ARRAY = 20; + } + optional PPhysicalOperator operator = 1; + repeated PValue children = 2; +} + +message PVersionValue { + optional string base_alias = 1; +} + +message PWindowedValue { + repeated PValue partitioning_values = 1; + repeated PValue argument_values = 2; +} + +message PCollateValue { + optional string collator_registry = 1; + optional PValue string_child = 2; + optional PValue locale_child = 3; + optional PValue strength_child = 4; +} + +enum PComparisonType { + EQUALS = 1; + NOT_EQUALS = 2; + LESS_THAN = 3; + LESS_THAN_OR_EQUALS = 4; + GREATER_THAN = 5; + GREATER_THAN_OR_EQUALS = 6; + STARTS_WITH = 7; + NOT_NULL = 8; + IS_NULL = 9; + IN = 10; + TEXT_CONTAINS_ALL = 11; + TEXT_CONTAINS_ALL_WITHIN = 12; + TEXT_CONTAINS_ANY = 13; + TEXT_CONTAINS_PHRASE = 14; + TEXT_CONTAINS_PREFIX = 15; + TEXT_CONTAINS_ALL_PREFIXES = 16; + TEXT_CONTAINS_ANY_PREFIX = 17; + SORT = 18; + LIKE = 19; +} + +enum PTupleSource { + KEY = 1; + VALUE = 2; + OTHER = 3; +} + +message PEnumLightValue { + optional string name = 1; + optional int32 number = 2; +} + +message PUUID { + optional uint64 mostSigBits = 1; + optional uint64 leastSigBits = 2; +} + +message PFDBRecordVersion { + optional bytes raw_bytes = 1; +} + +message PComparableObject { + oneof specific_object { + Value primitive_object = 1; + PEnumLightValue enum_object = 2; + PUUID uuid = 3; + PFDBRecordVersion fdb_record_version = 4; + bytes bytes_as_byte_string = 5; + } +} + +message Value { + // TODO: This should be a oneof {} + optional double double_value = 1; + optional float float_value = 2; + optional int64 long_value = 3; + optional bool bool_value = 4; + optional string string_value = 5; + optional bytes bytes_value = 6; + optional int32 int_value = 7; +} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidatorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidatorTest.java index 236079966f..d333c81f94 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidatorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidatorTest.java @@ -1020,7 +1020,7 @@ public void recordTypeKeyChanged() { RecordMetaDataProto.MetaData.Builder protoBuilder = metaData1.toProto().toBuilder() .setVersion(metaData1.getVersion() + 1); protoBuilder.getRecordTypesBuilder(0) - .setExplicitKey(RecordMetaDataProto.Value.newBuilder() + .setExplicitKey(com.apple.foundationdb.record.planprotos.Value.newBuilder() .setStringValue("new_key")); RecordMetaData metaData2 = RecordMetaData.build(protoBuilder.build()); assertInvalid("record type key changed", metaData1, metaData2); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java index f9443d8511..a59a35d24f 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java @@ -69,7 +69,7 @@ public void serializationTest(@Nonnull Object value) throws InvalidProtocolBuffe final LiteralKeyExpression keyExpression = Key.Expressions.value(value); final LiteralKeyExpression parsedViaProto = LiteralKeyExpression.fromProto(keyExpression.toProto()); final LiteralKeyExpression parsedViaBytes = LiteralKeyExpression.fromProto( - RecordMetaDataProto.Value.parseFrom(keyExpression.toProto().toByteArray())); + com.apple.foundationdb.record.planprotos.Value.parseFrom(keyExpression.toProto().toByteArray())); assertEquals(keyExpression, parsedViaProto); assertEquals(keyExpression, parsedViaBytes); if (value instanceof byte[]) { @@ -90,7 +90,7 @@ public void incorrectUnicodeSurrogatePairSerializationTest() throws InvalidProto final LiteralKeyExpression parsedViaProto = LiteralKeyExpression.fromProto(keyExpression.toProto()); final LiteralKeyExpression parsedViaBytes = LiteralKeyExpression.fromProto( - RecordMetaDataProto.Value.parseFrom(keyExpression.toProto().toByteArray())); + com.apple.foundationdb.record.planprotos.Value.parseFrom(keyExpression.toProto().toByteArray())); assertEquals(keyExpression, parsedViaProto); // Comparison uses proto objects, so both sides have Longs. // Comparison with proto objects works since we never leave Java. From 25eb6cfc7afce951cd6623f9ba5d71047b6840cf Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 12 Dec 2024 14:07:01 -0800 Subject: [PATCH 07/41] save --- .../apple/foundationdb/record/RecordMetaDataBuilder.java | 2 +- .../java/com/apple/foundationdb/record/metadata/UDF.java | 9 ++++++++- .../src/main/proto/record_metadata.proto | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index 1efaee08b9..8289e5cac4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -232,7 +232,7 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, PlanHashable.CURRENT_FOR_CONTINUATION); for (RecordMetaDataProto.UDF udf: metaDataProto.getUdfsList()) { - udfMap.put(udf.getName(), new UDF(udf.getName(), Value.fromValueProto(serializationContext, udf.getValue()))); + udfMap.put(udf.getName(), new UDF(udf.getName(), Value.fromValueProto(serializationContext, udf.getValue()), Value.fromValueProto(serializationContext, udf.getArgumentValue()))); } if (metaDataProto.hasSplitLongRecords()) { splitLongRecords = metaDataProto.getSplitLongRecords(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java index bb2410b1d3..38f8e09439 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java @@ -32,10 +32,13 @@ public class UDF { @Nonnull private final String udfName; @Nonnull private final Value value; + @Nonnull + private final Value argumentValue; - public UDF(@Nonnull String udfName, @Nonnull Value value) { + public UDF(@Nonnull String udfName, @Nonnull Value value, @Nonnull Value argumentValue) { this.udfName = udfName; this.value = value; + this.argumentValue = argumentValue; } @Nonnull @@ -46,12 +49,16 @@ public String getUdfName() { @Nonnull public Value getValue() {return value;} + @Nonnull + public Value getArgumentValue() {return argumentValue;} + public RecordMetaDataProto.UDF toProto() { PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, PlanHashable.CURRENT_FOR_CONTINUATION); return RecordMetaDataProto.UDF.newBuilder() .setName(udfName) .setValue(value.toValueProto(serializationContext)) + .setArgumentValue(argumentValue.toValueProto(serializationContext)) .build(); } } diff --git a/fdb-record-layer-core/src/main/proto/record_metadata.proto b/fdb-record-layer-core/src/main/proto/record_metadata.proto index 7d04612b65..80af7e337e 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata.proto @@ -105,6 +105,7 @@ message MetaData { message UDF { optional string name = 1; optional com.apple.foundationdb.record.planprotos.PValue value = 2; + optional com.apple.foundationdb.record.planprotos.PValue argument_value = 3; } message Then { From 6ebd19b3c178fc1e68cf6f58a4420bec713b7f18 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 12 Dec 2024 19:30:05 -0800 Subject: [PATCH 08/41] reformat --- .../apple/foundationdb/record/RecordMetaDataBuilder.java | 6 ------ .../java/com/apple/foundationdb/record/metadata/UDF.java | 1 + 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index 8289e5cac4..77faa58f04 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -56,7 +56,6 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -1192,14 +1191,9 @@ public void addFormerIndex(@Nonnull FormerIndex formerIndex) { } public void addUDF(@Nonnull UDF udf) { - // (TODO): check udfMap.put(udf.getUdfName(), udf); } - public void addUDFs(@Nonnull Collection udfList) { - udfList.forEach(this::addUDF); - } - public boolean isSplitLongRecords() { return splitLongRecords; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java index 38f8e09439..4684f7195b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java @@ -52,6 +52,7 @@ public String getUdfName() { @Nonnull public Value getArgumentValue() {return argumentValue;} + @Nonnull public RecordMetaDataProto.UDF toProto() { PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, PlanHashable.CURRENT_FOR_CONTINUATION); From f8088f56a3e930786751e1682c6951ced89060f6 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Sun, 15 Dec 2024 21:10:12 -0800 Subject: [PATCH 09/41] replace --- .../record/RecordMetaDataBuilder.java | 7 +- .../foundationdb/record/metadata/UDF.java | 17 +- .../plan/cascades/values/FunctionValue.java | 178 ++++++++++++++++++ .../src/main/proto/record_metadata.proto | 3 +- .../src/main/proto/record_value.proto | 8 + 5 files changed, 198 insertions(+), 15 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FunctionValue.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index 77faa58f04..611832839e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -56,6 +56,7 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -231,7 +232,7 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, PlanHashable.CURRENT_FOR_CONTINUATION); for (RecordMetaDataProto.UDF udf: metaDataProto.getUdfsList()) { - udfMap.put(udf.getName(), new UDF(udf.getName(), Value.fromValueProto(serializationContext, udf.getValue()), Value.fromValueProto(serializationContext, udf.getArgumentValue()))); + udfMap.put(udf.getName(), new UDF(udf.getName(), Value.fromValueProto(serializationContext, udf.getFunctionValue()))); } if (metaDataProto.hasSplitLongRecords()) { splitLongRecords = metaDataProto.getSplitLongRecords(); @@ -1194,6 +1195,10 @@ public void addUDF(@Nonnull UDF udf) { udfMap.put(udf.getUdfName(), udf); } + public void addUDFs(@Nonnull Collection udfs) { + udfs.forEach(udf -> udfMap.put(udf.getUdfName(), udf)); + } + public boolean isSplitLongRecords() { return splitLongRecords; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java index 4684f7195b..f1d9104d79 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java @@ -31,14 +31,11 @@ public class UDF { @Nonnull private final String udfName; @Nonnull - private final Value value; - @Nonnull - private final Value argumentValue; + private final Value functionValue; - public UDF(@Nonnull String udfName, @Nonnull Value value, @Nonnull Value argumentValue) { + public UDF(@Nonnull String udfName, @Nonnull Value functionValue) { this.udfName = udfName; - this.value = value; - this.argumentValue = argumentValue; + this.functionValue = functionValue; } @Nonnull @@ -47,10 +44,7 @@ public String getUdfName() { } @Nonnull - public Value getValue() {return value;} - - @Nonnull - public Value getArgumentValue() {return argumentValue;} + public Value getValue() {return functionValue;} @Nonnull public RecordMetaDataProto.UDF toProto() { @@ -58,8 +52,7 @@ public RecordMetaDataProto.UDF toProto() { PlanHashable.CURRENT_FOR_CONTINUATION); return RecordMetaDataProto.UDF.newBuilder() .setName(udfName) - .setValue(value.toValueProto(serializationContext)) - .setArgumentValue(argumentValue.toValueProto(serializationContext)) + .setFunctionValue(functionValue.toValueProto(serializationContext)) .build(); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FunctionValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FunctionValue.java new file mode 100644 index 0000000000..1b746fa715 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FunctionValue.java @@ -0,0 +1,178 @@ +/* + * FunctionValue.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.planprotos.PFunctionValue; +import com.apple.foundationdb.record.planprotos.PValue; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.google.auto.service.AutoService; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +public class FunctionValue extends AbstractValue{ + @Nonnull + private final String functionName; + + @Nonnull + private final Iterable children; // QuantifiedObjectValue + + @Nonnull + private final Value underlying; + + @Nonnull + private final Type resultType; + + private FunctionValue(@Nonnull String functionName, @Nonnull final Iterable children, @Nonnull final Value underlying, @Nonnull final Type resultType) { + this.functionName = functionName; + this.children = children; + this.underlying = underlying; + this.resultType = resultType; + } + + @Override + public int planHash(@Nonnull final PlanHashMode hashKind) { + return PlanHashable.objectsPlanHash(hashKind, this.getClass().getCanonicalName(), children); + } + + @Nonnull + @Override + protected Iterable computeChildren() { + return children; + } + + @Override + @Nonnull + public Type getResultType() { + return resultType; + } + + @Nonnull + public Value getUnderlying() { + return underlying; + } + + @Nullable + @Override + public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { + return call(StreamSupport.stream(children.spliterator(), false).map(c -> c.eval(store, context)).collect(Collectors.toList())); + } + + @Nonnull + @Override + public Value withChildren(Iterable newChildren) { + return new FunctionValue(functionName, newChildren, underlying, resultType); + } + + @Nonnull + public static FunctionValue of(@Nonnull String functionName, @Nonnull final Iterable children, @Nonnull final Value underlying, @Nonnull final Type resultType) { + return new FunctionValue(functionName, children, underlying, resultType); + } + + @Nullable + public Value call(@Nonnull List arguments) { + // replace the children in underlying with arguments + /* + TranslationMap.Builder translationMapBuilder = new TranslationMap.Builder(); + // assert arguments size = children size + var argumentIter = arguments.iterator(); + for (Value v: getChildren()) { + translationMapBuilder.when(((QuantifiedObjectValue)v).getAlias()).then((sourceAlias, leafValue) -> (Value)argumentIter.next()); + } + return translateCorrelations(translationMapBuilder.build()); + */ + return underlying.replace((v) -> { + if ((v instanceof QuantifiedObjectValue) && (v.getResultType().equals(resultType))) { + return (Value)arguments.get(0); + } else { + return v; + } + }); + } + + @Override + public int hashCodeWithoutChildren() { + return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, this.getClass().getCanonicalName()); + } + + @Nonnull + @Override + public PFunctionValue toProto(@Nonnull final PlanSerializationContext serializationContext) { + List childValueList = new LinkedList<>(); + children.forEach(c -> childValueList.add(c.toValueProto(serializationContext))); + PFunctionValue.Builder builder = PFunctionValue.newBuilder(); + builder.setFunctionName(functionName); + builder.setResultType(resultType.toTypeProto(serializationContext)); + builder.addAllChildren(childValueList); + builder.setUnderlying(underlying.toValueProto(serializationContext)); + return builder.build(); + } + + + @Nonnull + @Override + public PValue toValueProto(@Nonnull final PlanSerializationContext serializationContext) { + final var specificValueProto = toProto(serializationContext); + return PValue.newBuilder().setFunctionValue(specificValueProto).build(); + } + + @Nonnull + public static FunctionValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PFunctionValue functionValue) { + List children = new ArrayList<>(); + for (PValue pValue: functionValue.getChildrenList()) { + children.add(Value.fromValueProto(serializationContext, pValue)); + } + return new FunctionValue(functionValue.getFunctionName(), children, Value.fromValueProto(serializationContext, functionValue.getUnderlying()), Type.fromTypeProto(serializationContext, functionValue.getResultType())); + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PFunctionValue.class; + } + + @Nonnull + @Override + public FunctionValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PFunctionValue functionValue) { + return FunctionValue.fromProto(serializationContext, functionValue); + } + } +} + diff --git a/fdb-record-layer-core/src/main/proto/record_metadata.proto b/fdb-record-layer-core/src/main/proto/record_metadata.proto index 80af7e337e..bbf434c495 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata.proto @@ -104,8 +104,7 @@ message MetaData { message UDF { optional string name = 1; - optional com.apple.foundationdb.record.planprotos.PValue value = 2; - optional com.apple.foundationdb.record.planprotos.PValue argument_value = 3; + optional com.apple.foundationdb.record.planprotos.PValue function_value = 3; } message Then { diff --git a/fdb-record-layer-core/src/main/proto/record_value.proto b/fdb-record-layer-core/src/main/proto/record_value.proto index d2f43b1810..2bc87411f0 100644 --- a/fdb-record-layer-core/src/main/proto/record_value.proto +++ b/fdb-record-layer-core/src/main/proto/record_value.proto @@ -223,6 +223,7 @@ message PValue { PFromOrderedBytesValue from_ordered_bytes_value = 43; PCollateValue collate_value = 44; PNumericAggregationValue.PBitmapConstructAgg numeric_aggregation_value_bitmap_construct_agg = 45; + PFunctionValue function_value = 46; } } @@ -246,6 +247,13 @@ message PAndOrValue { optional POperator operator = 4; } +message PFunctionValue { + optional string function_name = 1; + repeated PValue children = 2; + optional PValue underlying = 3; + optional PType result_type = 4; +} + message PArithmeticValue { enum PPhysicalOperator { ADD_II = 1; From 157ab0c242d389f789521a1fe8de5dc9ab2c9639 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 18 Dec 2024 12:35:38 -0800 Subject: [PATCH 10/41] put keyexpression in separate proto --- .../record/RecordMetaDataBuilder.java | 6 +- .../foundationdb/record/metadata/Key.java | 15 +- .../foundationdb/record/metadata/UDF.java | 12 +- .../expressions/DimensionsKeyExpression.java | 12 +- .../expressions/EmptyKeyExpression.java | 12 +- .../expressions/FieldKeyExpression.java | 13 +- .../expressions/FunctionKeyExpression.java | 12 +- .../expressions/GroupingKeyExpression.java | 12 +- .../metadata/expressions/KeyExpression.java | 20 +- .../expressions/KeyWithValueExpression.java | 12 +- .../expressions/ListKeyExpression.java | 12 +- .../expressions/LiteralKeyExpression.java | 21 +- .../expressions/NestingKeyExpression.java | 12 +- .../expressions/RecordTypeKeyExpression.java | 12 +- .../expressions/SplitKeyExpression.java | 12 +- .../expressions/ThenKeyExpression.java | 14 +- .../expressions/VersionKeyExpression.java | 12 +- .../record/query/expressions/Comparisons.java | 11 +- .../plan/IndexKeyValueToPartialRecord.java | 11 +- .../KeyExpressionExpansionVisitor.java | 10 +- .../plan/cascades/NullableArrayTypeUtils.java | 9 +- .../plan/cascades/values/FunctionValue.java | 178 ---- .../cascades/values/MacroFunctionValue.java | 154 +++ .../main/proto/record_key_expression.proto | 118 +++ .../src/main/proto/record_metadata.proto | 94 +- .../main/proto/record_metadata_options.proto | 4 +- .../src/main/proto/record_query_plan.proto | 966 ++++++++++++++++- .../src/main/proto/record_query_runtime.proto | 3 +- .../src/main/proto/record_value.proto | 970 ------------------ .../MetaDataEvolutionValidatorTest.java | 3 +- .../record/metadata/MetaDataProtoTest.java | 41 +- .../record/metadata/UnknownKeyExpression.java | 3 +- .../expressions/LiteralKeyExpressionTest.java | 5 +- .../foundationdb/FDBMetaDataStoreTest.java | 13 +- .../foundationdb/FDBRecordStoreTest.java | 3 +- .../FDBRecordStoreStateCacheTest.java | 4 +- .../main/proto/lucene_record_query_plan.proto | 2 +- 37 files changed, 1411 insertions(+), 1412 deletions(-) delete mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FunctionValue.java create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java create mode 100644 fdb-record-layer-core/src/main/proto/record_key_expression.proto delete mode 100644 fdb-record-layer-core/src/main/proto/record_value.proto diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index 611832839e..d35d2e5acf 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -232,7 +232,7 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, PlanHashable.CURRENT_FOR_CONTINUATION); for (RecordMetaDataProto.UDF udf: metaDataProto.getUdfsList()) { - udfMap.put(udf.getName(), new UDF(udf.getName(), Value.fromValueProto(serializationContext, udf.getFunctionValue()))); + udfMap.put(udf.getFunctionName(), new UDF(udf.getFunctionName(), Value.fromValueProto(serializationContext, udf.getFunctionValue()))); } if (metaDataProto.hasSplitLongRecords()) { splitLongRecords = metaDataProto.getSplitLongRecords(); @@ -1192,11 +1192,11 @@ public void addFormerIndex(@Nonnull FormerIndex formerIndex) { } public void addUDF(@Nonnull UDF udf) { - udfMap.put(udf.getUdfName(), udf); + udfMap.put(udf.getFunctionName(), udf); } public void addUDFs(@Nonnull Collection udfs) { - udfs.forEach(udf -> udfMap.put(udf.getUdfName(), udf)); + udfs.forEach(udf -> udfMap.put(udf.getFunctionName(), udf)); } public boolean isSplitLongRecords() { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java index c175ee2545..8c9d75f29f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression; @@ -392,21 +393,21 @@ public static class Evaluated { * Values used in index keys in place of missing fields. */ public enum NullStandin { - NULL(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE), // Missing field here skips uniqueness checks. - NULL_UNIQUE(RecordMetaDataProto.Field.NullInterpretation.UNIQUE), // Missing field here like ordinary value, but null, for uniqueness. - NOT_NULL(RecordMetaDataProto.Field.NullInterpretation.NOT_NULL); // Missing field has type's ordinary default value. + NULL(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE), // Missing field here skips uniqueness checks. + NULL_UNIQUE(RecordKeyExpressionProto.Field.NullInterpretation.UNIQUE), // Missing field here like ordinary value, but null, for uniqueness. + NOT_NULL(RecordKeyExpressionProto.Field.NullInterpretation.NOT_NULL); // Missing field has type's ordinary default value. - private RecordMetaDataProto.Field.NullInterpretation proto; + private RecordKeyExpressionProto.Field.NullInterpretation proto; - NullStandin(RecordMetaDataProto.Field.NullInterpretation nullInterpretation) { + NullStandin(RecordKeyExpressionProto.Field.NullInterpretation nullInterpretation) { proto = nullInterpretation; } - public RecordMetaDataProto.Field.NullInterpretation toProto() { + public RecordKeyExpressionProto.Field.NullInterpretation toProto() { return proto; } - public static NullStandin valueOf(RecordMetaDataProto.Field.NullInterpretation nullInterpretation) throws KeyExpression.DeserializationException { + public static NullStandin valueOf(RecordKeyExpressionProto.Field.NullInterpretation nullInterpretation) throws KeyExpression.DeserializationException { switch (nullInterpretation) { case NOT_UNIQUE: return NULL; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java index f1d9104d79..ecd1b9cca7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java @@ -29,18 +29,18 @@ import javax.annotation.Nonnull; public class UDF { - @Nonnull private final String udfName; + @Nonnull private final String functionName; @Nonnull private final Value functionValue; - public UDF(@Nonnull String udfName, @Nonnull Value functionValue) { - this.udfName = udfName; + public UDF(@Nonnull String functionName, @Nonnull Value functionValue) { + this.functionName = functionName; this.functionValue = functionValue; } @Nonnull - public String getUdfName() { - return udfName; + public String getFunctionName() { + return functionName; } @Nonnull @@ -51,7 +51,7 @@ public RecordMetaDataProto.UDF toProto() { PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, PlanHashable.CURRENT_FOR_CONTINUATION); return RecordMetaDataProto.UDF.newBuilder() - .setName(udfName) + .setFunctionName(functionName) .setFunctionValue(functionValue.toValueProto(serializationContext)) .build(); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java index add3ac1dc5..30a74bcbe3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; @@ -63,7 +63,7 @@ private DimensionsKeyExpression(@Nonnull final KeyExpression wholeKey, this.dimensionsSize = dimensionsSize; } - DimensionsKeyExpression(@Nonnull final RecordMetaDataProto.Dimensions dimensions) throws DeserializationException { + DimensionsKeyExpression(@Nonnull final RecordKeyExpressionProto.Dimensions dimensions) throws DeserializationException { this(KeyExpression.fromProto(dimensions.getWholeKey()), dimensions.getPrefixSize(), dimensions.getDimensionsSize()); } @@ -114,8 +114,8 @@ public boolean needsCopyingToPartialRecord() { @Nonnull @Override - public RecordMetaDataProto.Dimensions toProto() throws SerializationException { - final RecordMetaDataProto.Dimensions.Builder builder = RecordMetaDataProto.Dimensions.newBuilder(); + public RecordKeyExpressionProto.Dimensions toProto() throws SerializationException { + final RecordKeyExpressionProto.Dimensions.Builder builder = RecordKeyExpressionProto.Dimensions.newBuilder(); builder.setWholeKey(getWholeKey().toKeyExpression()); builder.setPrefixSize(prefixSize); builder.setDimensionsSize(dimensionsSize); @@ -124,8 +124,8 @@ public RecordMetaDataProto.Dimensions toProto() throws SerializationException { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toKeyExpression() { - return RecordMetaDataProto.KeyExpression.newBuilder().setDimensions(toProto()).build(); + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { + return RecordKeyExpressionProto.KeyExpression.newBuilder().setDimensions(toProto()).build(); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java index f02581d3af..4068d36946 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; @@ -45,8 +45,8 @@ public class EmptyKeyExpression extends BaseKeyExpression implements KeyExpressi private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Empty-Key-Expression"); public static final EmptyKeyExpression EMPTY = new EmptyKeyExpression(); - public static final RecordMetaDataProto.KeyExpression EMPTY_PROTO = - RecordMetaDataProto.KeyExpression.newBuilder().setEmpty(EMPTY.toProto()).build(); + public static final RecordKeyExpressionProto.KeyExpression EMPTY_PROTO = + RecordKeyExpressionProto.KeyExpression.newBuilder().setEmpty(EMPTY.toProto()).build(); private EmptyKeyExpression() { // nothing to initialize @@ -75,8 +75,8 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.Empty toProto() throws SerializationException { - return RecordMetaDataProto.Empty.getDefaultInstance(); + public RecordKeyExpressionProto.Empty toProto() throws SerializationException { + return RecordKeyExpressionProto.Empty.getDefaultInstance(); } @Nonnull @@ -87,7 +87,7 @@ public R expand(@Nonnull final KeyExpr @Nonnull @Override - public RecordMetaDataProto.KeyExpression toKeyExpression() { + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { return EMPTY_PROTO; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java index 0ee180a576..d47e31dcca 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java @@ -24,7 +24,8 @@ import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; @@ -71,7 +72,7 @@ public FieldKeyExpression(@Nonnull String fieldName, @Nonnull FanType fanType, @ this.nullStandin = nullStandin; } - public FieldKeyExpression(@Nonnull RecordMetaDataProto.Field field) throws DeserializationException { + public FieldKeyExpression(@Nonnull RecordKeyExpressionProto.Field field) throws DeserializationException { if (!field.hasFieldName()) { throw new DeserializationException("Serialized Field is missing field name"); } @@ -193,8 +194,8 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.Field toProto() throws SerializationException { - return RecordMetaDataProto.Field.newBuilder() + public RecordKeyExpressionProto.Field toProto() throws SerializationException { + return RecordKeyExpressionProto.Field.newBuilder() .setFieldName(fieldName) .setFanType(fanType.toProto()) .setNullInterpretation(nullStandin.toProto()) @@ -203,8 +204,8 @@ public RecordMetaDataProto.Field toProto() throws SerializationException { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toKeyExpression() { - return RecordMetaDataProto.KeyExpression.newBuilder().setField(toProto()).build(); + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { + return RecordKeyExpressionProto.KeyExpression.newBuilder().setField(toProto()).build(); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java index 4ab23d4fae..6194200f40 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java @@ -25,7 +25,7 @@ import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.RecordCoreArgumentException; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; @@ -232,7 +232,7 @@ public boolean equalsAtomic(AtomKeyExpression other) { * implementation, or the arguments provided are not suitable for the function. */ @Nonnull - public static FunctionKeyExpression fromProto(RecordMetaDataProto.Function function) throws DeserializationException { + public static FunctionKeyExpression fromProto(RecordKeyExpressionProto.Function function) throws DeserializationException { try { return create(function.getName(), KeyExpression.fromProto(function.getArguments())); } catch (RecordCoreException e) { @@ -242,8 +242,8 @@ public static FunctionKeyExpression fromProto(RecordMetaDataProto.Function funct @Nonnull @Override - public final RecordMetaDataProto.Function toProto() throws SerializationException { - RecordMetaDataProto.Function.Builder builder = RecordMetaDataProto.Function.newBuilder() + public final RecordKeyExpressionProto.Function toProto() throws SerializationException { + RecordKeyExpressionProto.Function.Builder builder = RecordKeyExpressionProto.Function.newBuilder() .setName(getName()); builder.setArguments(getArguments().toKeyExpression()); return builder.build(); @@ -251,8 +251,8 @@ public final RecordMetaDataProto.Function toProto() throws SerializationExceptio @Nonnull @Override - public final RecordMetaDataProto.KeyExpression toKeyExpression() { - return RecordMetaDataProto.KeyExpression.newBuilder().setFunction(toProto()).build(); + public final RecordKeyExpressionProto.KeyExpression toKeyExpression() { + return RecordKeyExpressionProto.KeyExpression.newBuilder().setFunction(toProto()).build(); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java index eebb1678db..916cf3fd0f 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; @@ -53,7 +53,7 @@ public GroupingKeyExpression(@Nonnull KeyExpression wholeKey, int groupedCount) this.groupedCount = groupedCount; } - public GroupingKeyExpression(@Nonnull RecordMetaDataProto.Grouping grouping) throws DeserializationException { + public GroupingKeyExpression(@Nonnull RecordKeyExpressionProto.Grouping grouping) throws DeserializationException { this(KeyExpression.fromProto(grouping.getWholeKey()), grouping.getGroupedCount()); } @@ -100,8 +100,8 @@ public boolean needsCopyingToPartialRecord() { @Nonnull @Override - public RecordMetaDataProto.Grouping toProto() throws SerializationException { - final RecordMetaDataProto.Grouping.Builder builder = RecordMetaDataProto.Grouping.newBuilder(); + public RecordKeyExpressionProto.Grouping toProto() throws SerializationException { + final RecordKeyExpressionProto.Grouping.Builder builder = RecordKeyExpressionProto.Grouping.newBuilder(); builder.setWholeKey(getWholeKey().toKeyExpression()); builder.setGroupedCount(groupedCount); return builder.build(); @@ -109,8 +109,8 @@ public RecordMetaDataProto.Grouping toProto() throws SerializationException { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toKeyExpression() { - return RecordMetaDataProto.KeyExpression.newBuilder().setGrouping(toProto()).build(); + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { + return RecordKeyExpressionProto.KeyExpression.newBuilder().setGrouping(toProto()).build(); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java index ca3c01f0ef..06caf442de 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java @@ -24,7 +24,7 @@ import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.QueryHashable; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.ExpansionVisitor; @@ -165,28 +165,28 @@ enum FanType { * Convert a repeated field into a single list. * This does not cause the number of index values to increase. */ - Concatenate(RecordMetaDataProto.Field.FanType.CONCATENATE), + Concatenate(RecordKeyExpressionProto.Field.FanType.CONCATENATE), /** * Create an index value for each value of the field. */ - FanOut(RecordMetaDataProto.Field.FanType.FAN_OUT), + FanOut(RecordKeyExpressionProto.Field.FanType.FAN_OUT), /** * Nothing, only allowed with scalar fields. */ - None(RecordMetaDataProto.Field.FanType.SCALAR); + None(RecordKeyExpressionProto.Field.FanType.SCALAR); - private RecordMetaDataProto.Field.FanType proto; + private RecordKeyExpressionProto.Field.FanType proto; - FanType(RecordMetaDataProto.Field.FanType fanType) { + FanType(RecordKeyExpressionProto.Field.FanType fanType) { proto = fanType; } - RecordMetaDataProto.Field.FanType toProto() { + RecordKeyExpressionProto.Field.FanType toProto() { return proto; } - public static FanType valueOf(RecordMetaDataProto.Field.FanType fanType) throws DeserializationException { + public static FanType valueOf(RecordKeyExpressionProto.Field.FanType fanType) throws DeserializationException { switch (fanType) { case SCALAR: return None; @@ -204,7 +204,7 @@ public static FanType valueOf(RecordMetaDataProto.Field.FanType fanType) throws Message toProto() throws SerializationException; @Nonnull - RecordMetaDataProto.KeyExpression toKeyExpression(); + RecordKeyExpressionProto.KeyExpression toKeyExpression(); /** * Get key in normalized form for comparing field-by-field. @@ -339,7 +339,7 @@ default boolean hasProperInterfaces() { } @Nonnull - static KeyExpression fromProto(RecordMetaDataProto.KeyExpression expression) + static KeyExpression fromProto(RecordKeyExpressionProto.KeyExpression expression) throws DeserializationException { KeyExpression root = null; int found = 0; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java index ea224bc67e..37bd7acda4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; @@ -61,7 +61,7 @@ public KeyWithValueExpression(@Nonnull KeyExpression innerKey, int splitPoint) { this.splitPoint = splitPoint; } - public KeyWithValueExpression(@Nonnull RecordMetaDataProto.KeyWithValue proto) throws DeserializationException { + public KeyWithValueExpression(@Nonnull RecordKeyExpressionProto.KeyWithValue proto) throws DeserializationException { this(KeyExpression.fromProto(proto.getInnerKey()), proto.getSplitPoint()); } @@ -133,8 +133,8 @@ public boolean needsCopyingToPartialRecord() { @Nonnull @Override - public RecordMetaDataProto.KeyWithValue toProto() throws SerializationException { - return RecordMetaDataProto.KeyWithValue.newBuilder() + public RecordKeyExpressionProto.KeyWithValue toProto() throws SerializationException { + return RecordKeyExpressionProto.KeyWithValue.newBuilder() .setInnerKey(getInnerKey().toKeyExpression()) .setSplitPoint(splitPoint) .build(); @@ -142,8 +142,8 @@ public RecordMetaDataProto.KeyWithValue toProto() throws SerializationException @Nonnull @Override - public RecordMetaDataProto.KeyExpression toKeyExpression() { - return RecordMetaDataProto.KeyExpression.newBuilder().setKeyWithValue(toProto()).build(); + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { + return RecordKeyExpressionProto.KeyExpression.newBuilder().setKeyWithValue(toProto()).build(); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java index 8552719078..0a71db1f51 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; @@ -79,7 +79,7 @@ private ListKeyExpression(@Nonnull ListKeyExpression orig, int start, int end) { children = orig.children.subList(start, end); } - public ListKeyExpression(@Nonnull RecordMetaDataProto.List list) throws DeserializationException { + public ListKeyExpression(@Nonnull RecordKeyExpressionProto.List list) throws DeserializationException { children = list.getChildList().stream().map(KeyExpression::fromProto).collect(Collectors.toList()); } @@ -140,8 +140,8 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.List toProto() throws SerializationException { - final RecordMetaDataProto.List.Builder builder = RecordMetaDataProto.List.newBuilder(); + public RecordKeyExpressionProto.List toProto() throws SerializationException { + final RecordKeyExpressionProto.List.Builder builder = RecordKeyExpressionProto.List.newBuilder(); for (KeyExpression child : children) { builder.addChild(child.toKeyExpression()); } @@ -150,8 +150,8 @@ public RecordMetaDataProto.List toProto() throws SerializationException { @Override @Nonnull - public RecordMetaDataProto.KeyExpression toKeyExpression() { - return RecordMetaDataProto.KeyExpression.newBuilder().setList(toProto()).build(); + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { + return RecordKeyExpressionProto.KeyExpression.newBuilder().setList(toProto()).build(); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java index 8cf2d8ea26..01e661c067 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java @@ -24,9 +24,8 @@ import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; -import com.apple.foundationdb.record.planprotos.RecordMetaDataValueProto; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; @@ -58,14 +57,14 @@ public class LiteralKeyExpression extends BaseKeyExpression implements AtomKe @Nonnull private final List evaluated; @Nonnull - private final com.apple.foundationdb.record.planprotos.Value proto; + private final RecordKeyExpressionProto.Value proto; public LiteralKeyExpression(@Nullable T value) { // getProto() performs validation that it is a type we can serialize this(value, toProtoValue(value)); } - private LiteralKeyExpression(@Nullable T value, @Nonnull com.apple.foundationdb.record.planprotos.Value proto) { + private LiteralKeyExpression(@Nullable T value, @Nonnull RecordKeyExpressionProto.Value proto) { this.value = value; this.evaluated = ImmutableList.of(value == null ? Key.Evaluated.NULL : Key.Evaluated.scalar(value)); this.proto = proto; @@ -99,7 +98,7 @@ public int getColumnSize() { @Nonnull @Override - public com.apple.foundationdb.record.planprotos.Value toProto() throws SerializationException { + public RecordKeyExpressionProto.Value toProto() throws SerializationException { return proto; } @@ -118,8 +117,8 @@ public Value toValue(@Nonnull final CorrelationIdentifier baseAlias, @Nonnull @Override - public RecordMetaDataProto.KeyExpression toKeyExpression() { - return RecordMetaDataProto.KeyExpression.newBuilder().setValue(toProto()).build(); + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { + return RecordKeyExpressionProto.KeyExpression.newBuilder().setValue(toProto()).build(); } @Override @@ -128,12 +127,12 @@ public boolean needsCopyingToPartialRecord() { } @Nonnull - public static LiteralKeyExpression fromProto(com.apple.foundationdb.record.planprotos.Value proto) { + public static LiteralKeyExpression fromProto(RecordKeyExpressionProto.Value proto) { return new LiteralKeyExpression<>(fromProtoValue(proto), proto); } @Nullable - public static Object fromProtoValue(com.apple.foundationdb.record.planprotos.Value proto) { + public static Object fromProtoValue(RecordKeyExpressionProto.Value proto) { int found = 0; Object value = null; if (proto.hasDoubleValue()) { @@ -175,8 +174,8 @@ public static Object fromProtoValue(com.apple.foundationdb.record.planprotos.Val } @Nonnull - public static com.apple.foundationdb.record.planprotos.Value toProtoValue(@Nullable Object value) { - com.apple.foundationdb.record.planprotos.Value.Builder builder = com.apple.foundationdb.record.planprotos.Value.newBuilder(); + public static RecordKeyExpressionProto.Value toProtoValue(@Nullable Object value) { + RecordKeyExpressionProto.Value.Builder builder = RecordKeyExpressionProto.Value.newBuilder(); if (value instanceof Double) { builder.setDoubleValue((Double) value); } else if (value instanceof Float) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java index 6b0e823ae5..321c3fa133 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; @@ -60,7 +60,7 @@ public NestingKeyExpression(@Nonnull FieldKeyExpression parent, @Nonnull KeyExpr this.child = child; } - public NestingKeyExpression(@Nonnull RecordMetaDataProto.Nesting nesting) throws DeserializationException { + public NestingKeyExpression(@Nonnull RecordKeyExpressionProto.Nesting nesting) throws DeserializationException { if (!nesting.hasParent()) { throw new DeserializationException("Serialized Nesting is missing parent"); } @@ -100,8 +100,8 @@ public int getColumnSize() { @Nonnull @Override - public RecordMetaDataProto.Nesting toProto() throws SerializationException { - final RecordMetaDataProto.Nesting.Builder builder = RecordMetaDataProto.Nesting.newBuilder(); + public RecordKeyExpressionProto.Nesting toProto() throws SerializationException { + final RecordKeyExpressionProto.Nesting.Builder builder = RecordKeyExpressionProto.Nesting.newBuilder(); builder.setParent(parent.toProto()); builder.setChild(getChild().toKeyExpression()); return builder.build(); @@ -109,8 +109,8 @@ public RecordMetaDataProto.Nesting toProto() throws SerializationException { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toKeyExpression() { - return RecordMetaDataProto.KeyExpression.newBuilder().setNesting(toProto()).build(); + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { + return RecordKeyExpressionProto.KeyExpression.newBuilder().setNesting(toProto()).build(); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java index 8bdb00eab7..abfc849f6e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; @@ -60,8 +60,8 @@ public class RecordTypeKeyExpression extends BaseKeyExpression implements AtomKeyExpression, KeyExpressionWithoutChildren, KeyExpressionWithValue { private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Record-Type-Key-Expression"); public static final RecordTypeKeyExpression RECORD_TYPE_KEY = new RecordTypeKeyExpression(); - public static final RecordMetaDataProto.KeyExpression RECORD_TYPE_KEY_PROTO = - RecordMetaDataProto.KeyExpression.newBuilder().setRecordTypeKey(RECORD_TYPE_KEY.toProto()).build(); + public static final RecordKeyExpressionProto.KeyExpression RECORD_TYPE_KEY_PROTO = + RecordKeyExpressionProto.KeyExpression.newBuilder().setRecordTypeKey(RECORD_TYPE_KEY.toProto()).build(); private static final GroupingKeyExpression UNGROUPED = new GroupingKeyExpression(new RecordTypeKeyExpression(), 0); @@ -107,13 +107,13 @@ public GroupingKeyExpression ungrouped() { @Nonnull @Override - public RecordMetaDataProto.RecordTypeKey toProto() throws SerializationException { - return RecordMetaDataProto.RecordTypeKey.getDefaultInstance(); + public RecordKeyExpressionProto.RecordTypeKey toProto() throws SerializationException { + return RecordKeyExpressionProto.RecordTypeKey.getDefaultInstance(); } @Nonnull @Override - public RecordMetaDataProto.KeyExpression toKeyExpression() { + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { return RECORD_TYPE_KEY_PROTO; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java index a669faa522..d015c84457 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java @@ -24,7 +24,7 @@ import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; @@ -56,7 +56,7 @@ public SplitKeyExpression(KeyExpression joined, int splitSize) { this.splitSize = splitSize; } - public SplitKeyExpression(@Nonnull RecordMetaDataProto.Split split) throws DeserializationException { + public SplitKeyExpression(@Nonnull RecordKeyExpressionProto.Split split) throws DeserializationException { this(KeyExpression.fromProto(split.getJoined()), split.getSplitSize()); } @@ -110,8 +110,8 @@ public boolean needsCopyingToPartialRecord() { @Nonnull @Override - public RecordMetaDataProto.Split toProto() throws SerializationException { - final RecordMetaDataProto.Split.Builder builder = RecordMetaDataProto.Split.newBuilder(); + public RecordKeyExpressionProto.Split toProto() throws SerializationException { + final RecordKeyExpressionProto.Split.Builder builder = RecordKeyExpressionProto.Split.newBuilder(); builder.setJoined(getJoined().toKeyExpression()); builder.setSplitSize(splitSize); return builder.build(); @@ -119,8 +119,8 @@ public RecordMetaDataProto.Split toProto() throws SerializationException { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toKeyExpression() { - return RecordMetaDataProto.KeyExpression.newBuilder().setSplit(toProto()).build(); + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { + return RecordKeyExpressionProto.KeyExpression.newBuilder().setSplit(toProto()).build(); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java index 776c48632f..69fc395d9c 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java @@ -24,7 +24,7 @@ import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; @@ -75,9 +75,9 @@ public ThenKeyExpression(@Nonnull KeyExpression first, @Nonnull KeyExpression se } } - public ThenKeyExpression(@Nonnull RecordMetaDataProto.Then then) throws DeserializationException { + public ThenKeyExpression(@Nonnull RecordKeyExpressionProto.Then then) throws DeserializationException { children = new ArrayList<>(then.getChildCount()); - for (RecordMetaDataProto.KeyExpression child : then.getChildList()) { + for (RecordKeyExpressionProto.KeyExpression child : then.getChildList()) { final KeyExpression expression = KeyExpression.fromProto(child); add(children, expression); } @@ -171,8 +171,8 @@ public GroupingKeyExpression group(int count) { @Nonnull @Override - public RecordMetaDataProto.Then toProto() throws SerializationException { - final RecordMetaDataProto.Then.Builder builder = RecordMetaDataProto.Then.newBuilder(); + public RecordKeyExpressionProto.Then toProto() throws SerializationException { + final RecordKeyExpressionProto.Then.Builder builder = RecordKeyExpressionProto.Then.newBuilder(); for (KeyExpression child : children) { builder.addChild(child.toKeyExpression()); } @@ -181,8 +181,8 @@ public RecordMetaDataProto.Then toProto() throws SerializationException { @Override @Nonnull - public RecordMetaDataProto.KeyExpression toKeyExpression() { - return RecordMetaDataProto.KeyExpression.newBuilder().setThen(toProto()).build(); + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { + return RecordKeyExpressionProto.KeyExpression.newBuilder().setThen(toProto()).build(); } @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java index 632f269cd7..3859a78f6c 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion; @@ -49,8 +49,8 @@ @API(API.Status.MAINTAINED) public class VersionKeyExpression extends BaseKeyExpression implements AtomKeyExpression, KeyExpressionWithoutChildren, KeyExpressionWithValue { public static final VersionKeyExpression VERSION = new VersionKeyExpression(); - public static final RecordMetaDataProto.KeyExpression VERSION_PROTO = - RecordMetaDataProto.KeyExpression.newBuilder().setVersion(VERSION.toProto()).build(); + public static final RecordKeyExpressionProto.KeyExpression VERSION_PROTO = + RecordKeyExpressionProto.KeyExpression.newBuilder().setVersion(VERSION.toProto()).build(); private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Version-Key-Expression"); private static final GroupingKeyExpression UNGROUPED = new GroupingKeyExpression(new VersionKeyExpression(), 1); @@ -97,13 +97,13 @@ public GroupingKeyExpression groupBy(@Nonnull KeyExpression groupByFirst, @Nonnu @Nonnull @Override - public RecordMetaDataProto.Version toProto() throws SerializationException { - return RecordMetaDataProto.Version.getDefaultInstance(); + public RecordKeyExpressionProto.Version toProto() throws SerializationException { + return RecordKeyExpressionProto.Version.getDefaultInstance(); } @Nonnull @Override - public RecordMetaDataProto.KeyExpression toKeyExpression() { + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { return VERSION_PROTO; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java index d289ab1254..6e30fa1b40 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java @@ -38,7 +38,6 @@ import com.apple.foundationdb.record.metadata.expressions.InvertibleFunctionKeyExpression; import com.apple.foundationdb.record.metadata.expressions.TupleFieldsHelper; import com.apple.foundationdb.record.planprotos.PComparison; -import com.apple.foundationdb.record.planprotos.PComparisonType; import com.apple.foundationdb.record.planprotos.PInvertedFunctionComparison; import com.apple.foundationdb.record.planprotos.PListComparison; import com.apple.foundationdb.record.planprotos.PMultiColumnComparison; @@ -632,8 +631,8 @@ public enum Type { LIKE; @Nonnull - private static final Supplier> protoEnumBiMapSupplier = - Suppliers.memoize(() -> PlanSerialization.protoEnumBiMap(Type.class, PComparisonType.class)); + private static final Supplier> protoEnumBiMapSupplier = + Suppliers.memoize(() -> PlanSerialization.protoEnumBiMap(Type.class, PComparison.PComparisonType.class)); private final boolean isEquality; private final boolean isUnary; @@ -661,19 +660,19 @@ public boolean isUnary() { @Nonnull @SuppressWarnings("unused") - public PComparisonType toProto(@Nonnull final PlanSerializationContext serializationContext) { + public PComparison.PComparisonType toProto(@Nonnull final PlanSerializationContext serializationContext) { return Objects.requireNonNull(getProtoEnumBiMap().get(this)); } @Nonnull @SuppressWarnings("unused") public static Type fromProto(@Nonnull final PlanSerializationContext serializationContext, - @Nonnull final PComparisonType physicalOperatorProto) { + @Nonnull final PComparison.PComparisonType physicalOperatorProto) { return Objects.requireNonNull(getProtoEnumBiMap().inverse().get(physicalOperatorProto)); } @Nonnull - private static BiMap getProtoEnumBiMap() { + private static BiMap getProtoEnumBiMap() { return protoEnumBiMapSupplier.get(); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java index c7cd10c1f1..a68a73abbe 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java @@ -40,7 +40,6 @@ import com.apple.foundationdb.record.planprotos.PIndexKeyValueToPartialRecord.PFieldCopier; import com.apple.foundationdb.record.planprotos.PIndexKeyValueToPartialRecord.PFieldWithValueCopier; import com.apple.foundationdb.record.planprotos.PIndexKeyValueToPartialRecord.PMessageCopier; -import com.apple.foundationdb.record.planprotos.PTupleSource; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.predicates.ConstantPredicate; @@ -195,14 +194,14 @@ public enum TupleSource { @Nonnull @SuppressWarnings("unused") - public PTupleSource toProto(@Nonnull final PlanSerializationContext serializationContext) { + public PIndexKeyValueToPartialRecord.PTupleSource toProto(@Nonnull final PlanSerializationContext serializationContext) { switch (this) { case KEY: - return PTupleSource.KEY; + return PIndexKeyValueToPartialRecord.PTupleSource.KEY; case VALUE: - return PTupleSource.VALUE; + return PIndexKeyValueToPartialRecord.PTupleSource.VALUE; case OTHER: - return PTupleSource.OTHER; + return PIndexKeyValueToPartialRecord.PTupleSource.OTHER; default: throw new RecordCoreException("unknown tuple source mapping. did you forget to add it?"); } @@ -211,7 +210,7 @@ public PTupleSource toProto(@Nonnull final PlanSerializationContext serializatio @Nonnull @SuppressWarnings("unused") public static TupleSource fromProto(@Nonnull final PlanSerializationContext serializationContext, - @Nonnull final PTupleSource tupleSourceProto) { + @Nonnull final PIndexKeyValueToPartialRecord.PTupleSource tupleSourceProto) { switch (tupleSourceProto) { case KEY: return KEY; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java index 482fefd3b4..8cb1da0112 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java @@ -21,7 +21,7 @@ package com.apple.foundationdb.record.query.plan.cascades; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression; import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; import com.apple.foundationdb.record.metadata.expressions.FunctionKeyExpression; @@ -238,14 +238,14 @@ public GraphExpansion visitExpression(@Nonnull final NestingKeyExpression nestin .add(parent.getFieldName()) .build(); if (NullableArrayTypeUtils.isArrayWrapper(nestingKeyExpression)) { - final RecordMetaDataProto.KeyExpression childProto = nestingKeyExpression.getChild().toKeyExpression(); + final RecordKeyExpressionProto.KeyExpression childProto = nestingKeyExpression.getChild().toKeyExpression(); if (childProto.hasNesting()) { - RecordMetaDataProto.Nesting.Builder newNestingBuilder = RecordMetaDataProto.Nesting.newBuilder() - .setParent(parent.toProto().toBuilder().setFanType(RecordMetaDataProto.Field.FanType.FAN_OUT)) + RecordKeyExpressionProto.Nesting.Builder newNestingBuilder = RecordKeyExpressionProto.Nesting.newBuilder() + .setParent(parent.toProto().toBuilder().setFanType(RecordKeyExpressionProto.Field.FanType.FAN_OUT)) .setChild(childProto.getNesting().getChild()); return visitExpression(new NestingKeyExpression(newNestingBuilder.build())); } else { - return visitExpression(new FieldKeyExpression(parent.toProto().toBuilder().setFanType(RecordMetaDataProto.Field.FanType.FAN_OUT).build())); + return visitExpression(new FieldKeyExpression(parent.toProto().toBuilder().setFanType(RecordKeyExpressionProto.Field.FanType.FAN_OUT).build())); } } return pop(child.expand(push(state.withFieldNamePrefix(newPrefix)))); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/NullableArrayTypeUtils.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/NullableArrayTypeUtils.java index 14637de059..35bd7d5bef 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/NullableArrayTypeUtils.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/NullableArrayTypeUtils.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.record.query.plan.cascades; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.expressions.NestingKeyExpression; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; @@ -79,10 +80,10 @@ public static boolean describesWrappedArray(@Nonnull Descriptors.Descriptor desc * @return true if it describes a wrapped array, otherwise false. */ public static boolean isArrayWrapper(@Nonnull NestingKeyExpression nestingKeyExpression) { - RecordMetaDataProto.KeyExpression child = nestingKeyExpression.getChild().toKeyExpression(); + RecordKeyExpressionProto.KeyExpression child = nestingKeyExpression.getChild().toKeyExpression(); if (child.hasNesting()) { // if child is Nesting, check child.parent - RecordMetaDataProto.Field firstChild = child.getNesting().getParent(); + RecordKeyExpressionProto.Field firstChild = child.getNesting().getParent(); return isWrappedField(firstChild); } else if (child.hasField()) { // if child is Field, check itself @@ -118,7 +119,7 @@ public static Object unwrapIfArray(@Nullable Object wrappedValue, @Nonnull Type * * @return true if it is a wrapped array, otherwise false. */ - private static boolean isWrappedField(@Nonnull RecordMetaDataProto.Field field) { - return REPEATED_FIELD_NAME.equals(field.getFieldName()) && RecordMetaDataProto.Field.FanType.FAN_OUT.equals(field.getFanType()); + private static boolean isWrappedField(@Nonnull RecordKeyExpressionProto.Field field) { + return REPEATED_FIELD_NAME.equals(field.getFieldName()) && RecordKeyExpressionProto.Field.FanType.FAN_OUT.equals(field.getFanType()); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FunctionValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FunctionValue.java deleted file mode 100644 index 1b746fa715..0000000000 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FunctionValue.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * FunctionValue.java - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apple.foundationdb.record.query.plan.cascades.values; - -import com.apple.foundationdb.record.EvaluationContext; -import com.apple.foundationdb.record.PlanDeserializer; -import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.PlanSerializationContext; -import com.apple.foundationdb.record.planprotos.PFunctionValue; -import com.apple.foundationdb.record.planprotos.PValue; -import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; -import com.apple.foundationdb.record.query.plan.cascades.typing.Type; -import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; -import com.google.auto.service.AutoService; -import com.google.protobuf.Message; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -public class FunctionValue extends AbstractValue{ - @Nonnull - private final String functionName; - - @Nonnull - private final Iterable children; // QuantifiedObjectValue - - @Nonnull - private final Value underlying; - - @Nonnull - private final Type resultType; - - private FunctionValue(@Nonnull String functionName, @Nonnull final Iterable children, @Nonnull final Value underlying, @Nonnull final Type resultType) { - this.functionName = functionName; - this.children = children; - this.underlying = underlying; - this.resultType = resultType; - } - - @Override - public int planHash(@Nonnull final PlanHashMode hashKind) { - return PlanHashable.objectsPlanHash(hashKind, this.getClass().getCanonicalName(), children); - } - - @Nonnull - @Override - protected Iterable computeChildren() { - return children; - } - - @Override - @Nonnull - public Type getResultType() { - return resultType; - } - - @Nonnull - public Value getUnderlying() { - return underlying; - } - - @Nullable - @Override - public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { - return call(StreamSupport.stream(children.spliterator(), false).map(c -> c.eval(store, context)).collect(Collectors.toList())); - } - - @Nonnull - @Override - public Value withChildren(Iterable newChildren) { - return new FunctionValue(functionName, newChildren, underlying, resultType); - } - - @Nonnull - public static FunctionValue of(@Nonnull String functionName, @Nonnull final Iterable children, @Nonnull final Value underlying, @Nonnull final Type resultType) { - return new FunctionValue(functionName, children, underlying, resultType); - } - - @Nullable - public Value call(@Nonnull List arguments) { - // replace the children in underlying with arguments - /* - TranslationMap.Builder translationMapBuilder = new TranslationMap.Builder(); - // assert arguments size = children size - var argumentIter = arguments.iterator(); - for (Value v: getChildren()) { - translationMapBuilder.when(((QuantifiedObjectValue)v).getAlias()).then((sourceAlias, leafValue) -> (Value)argumentIter.next()); - } - return translateCorrelations(translationMapBuilder.build()); - */ - return underlying.replace((v) -> { - if ((v instanceof QuantifiedObjectValue) && (v.getResultType().equals(resultType))) { - return (Value)arguments.get(0); - } else { - return v; - } - }); - } - - @Override - public int hashCodeWithoutChildren() { - return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, this.getClass().getCanonicalName()); - } - - @Nonnull - @Override - public PFunctionValue toProto(@Nonnull final PlanSerializationContext serializationContext) { - List childValueList = new LinkedList<>(); - children.forEach(c -> childValueList.add(c.toValueProto(serializationContext))); - PFunctionValue.Builder builder = PFunctionValue.newBuilder(); - builder.setFunctionName(functionName); - builder.setResultType(resultType.toTypeProto(serializationContext)); - builder.addAllChildren(childValueList); - builder.setUnderlying(underlying.toValueProto(serializationContext)); - return builder.build(); - } - - - @Nonnull - @Override - public PValue toValueProto(@Nonnull final PlanSerializationContext serializationContext) { - final var specificValueProto = toProto(serializationContext); - return PValue.newBuilder().setFunctionValue(specificValueProto).build(); - } - - @Nonnull - public static FunctionValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PFunctionValue functionValue) { - List children = new ArrayList<>(); - for (PValue pValue: functionValue.getChildrenList()) { - children.add(Value.fromValueProto(serializationContext, pValue)); - } - return new FunctionValue(functionValue.getFunctionName(), children, Value.fromValueProto(serializationContext, functionValue.getUnderlying()), Type.fromTypeProto(serializationContext, functionValue.getResultType())); - } - - /** - * Deserializer. - */ - @AutoService(PlanDeserializer.class) - public static class Deserializer implements PlanDeserializer { - @Nonnull - @Override - public Class getProtoMessageClass() { - return PFunctionValue.class; - } - - @Nonnull - @Override - public FunctionValue fromProto(@Nonnull final PlanSerializationContext serializationContext, - @Nonnull final PFunctionValue functionValue) { - return FunctionValue.fromProto(serializationContext, functionValue); - } - } -} - diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java new file mode 100644 index 0000000000..cd30bc3a3a --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java @@ -0,0 +1,154 @@ +/* + * FunctionValue.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.planprotos.PMacroFunctionValue; +import com.apple.foundationdb.record.planprotos.PValue; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.cascades.SemanticException; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.stream.Collectors; + +public class MacroFunctionValue extends AbstractValue{ + + private final List argList; + + @Nonnull + private final Value body; + + private MacroFunctionValue(@Nonnull final List argList, @Nonnull final Value underlying) { + this.argList = argList; + this.body = underlying; + } + + @Override + public int planHash(@Nonnull final PlanHashMode hashKind) { + return PlanHashable.objectsPlanHash(hashKind, argList, body); + } + + @Override + @Nonnull + public Type getResultType() { + return body.getResultType(); + } + + @Nonnull + public Value getBody() { + return body; + } + + @Nullable + @Override + public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { + return body.eval(store, context); + } + + @Nonnull + public static MacroFunctionValue of(@Nonnull final List argList, @Nonnull final Value underlying, @Nonnull final Type resultType) { + return new MacroFunctionValue(argList, underlying); + } + + @Nullable + public Value call(@Nonnull List arguments) { + // replace the QuantifiedObjectValue in body with arguments + SemanticException.check(arguments.size() == argList.size(), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument length doesn't match with function definition"); + TranslationMap.Builder translationMapBuilder = new TranslationMap.Builder(); + for (int i = 0; i < arguments.size(); i++) { + // check if arguments[i] type matches with argList[i] type + final int finalI = i; + SemanticException.check(arguments.get(finalI) instanceof Value, SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "function input arguments must be of type Value"); + final var currentArg = (Value) arguments.get(finalI); + SemanticException.check(currentArg.getResultType().equals(argList.get(i).getResultType()), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument type doesn't match with function definition"); + translationMapBuilder.when(((QuantifiedObjectValue) argList.get(i)).getAlias()).then((sourceAlias, leafValue) -> (Value)arguments.get(finalI)); + } + return body.translateCorrelations(translationMapBuilder.build()); + } + + @Override + public int hashCodeWithoutChildren() { + return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, this.getClass().getCanonicalName()); + } + + @Nonnull + @Override + public PMacroFunctionValue toProto(@Nonnull final PlanSerializationContext serializationContext) { + PMacroFunctionValue.Builder builder = PMacroFunctionValue.newBuilder(); + argList.forEach(arg -> builder.addArgumentList(arg.toValueProto(serializationContext))); + return builder.setBody(body.toValueProto(serializationContext)).build(); + } + + + @Nonnull + @Override + public PValue toValueProto(@Nonnull final PlanSerializationContext serializationContext) { + final var specificValueProto = toProto(serializationContext); + return PValue.newBuilder().setMacroFunctionValue(specificValueProto).build(); + } + + @Nonnull + public static MacroFunctionValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PMacroFunctionValue functionValue) { + return new MacroFunctionValue(functionValue.getArgumentListList().stream().map(pvalue -> Value.fromValueProto(serializationContext, pvalue)).collect(Collectors.toList()), Value.fromValueProto(serializationContext, functionValue.getBody())); + } + + @Nonnull + @Override + protected Iterable computeChildren() { + return ImmutableList.of(); + } + + @Nonnull + @Override + public Value withChildren(final Iterable newChildren) { + return this; + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PMacroFunctionValue.class; + } + + @Nonnull + @Override + public MacroFunctionValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PMacroFunctionValue functionValue) { + return MacroFunctionValue.fromProto(serializationContext, functionValue); + } + } +} + diff --git a/fdb-record-layer-core/src/main/proto/record_key_expression.proto b/fdb-record-layer-core/src/main/proto/record_key_expression.proto new file mode 100644 index 0000000000..9c390e14b3 --- /dev/null +++ b/fdb-record-layer-core/src/main/proto/record_key_expression.proto @@ -0,0 +1,118 @@ +/* + * record_key_expression.proto + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2018 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +syntax = "proto2"; + +package com.apple.foundationdb.record; +import "google/protobuf/descriptor.proto"; + +option java_outer_classname = "RecordKeyExpressionProto"; + +message Then { + repeated KeyExpression child = 1; +} + +message List { + repeated KeyExpression child = 1; +} + +message Field { + required string field_name = 1; + required FanType fan_type = 2; + enum FanType { + SCALAR = 1; + FAN_OUT = 2; + CONCATENATE = 3; + } + optional NullInterpretation nullInterpretation = 3 [default = NOT_UNIQUE]; + enum NullInterpretation { + NOT_UNIQUE = 1; // Missing values allowed multiple times in unique index + UNIQUE = 2; // Missing values treated like null value in unique index + NOT_NULL = 3; // Missing values are the default for the type, not null + } +} + +message Nesting { + optional Field parent = 1; + optional KeyExpression child = 2; +} + +message Grouping { + required KeyExpression whole_key = 1; + optional int32 grouped_count = 3 [default = 1]; +} + +message Dimensions { + required KeyExpression whole_key = 1; + optional int32 prefix_size = 2; + optional int32 dimensions_size = 3; +} + +message KeyWithValue { + required KeyExpression inner_key = 1; + optional int32 split_point = 2 [default = 1]; +} + +message Split { + required KeyExpression joined = 1; + optional int32 split_size = 2; +} + +message Empty { +} + +message Version { +} + +message RecordTypeKey { +} + +message Value { + // TODO: This should be a oneof {} + optional double double_value = 1; + optional float float_value = 2; + optional int64 long_value = 3; + optional bool bool_value = 4; + optional string string_value = 5; + optional bytes bytes_value = 6; + optional int32 int_value = 7; +} + + +message Function { + required string name = 1; + required KeyExpression arguments = 2; +} + +message KeyExpression { + // Exactly one of the following: + optional Then then = 1; + optional Nesting nesting = 2; + optional Field field = 3; + optional Grouping grouping = 4; + optional Empty empty = 5; + optional Split split = 6; + optional Version version = 7; + optional Value value = 8; + optional Function function = 9; + optional KeyWithValue key_with_value = 10; + optional RecordTypeKey record_type_key = 11; + optional List list = 12; + optional Dimensions dimensions = 13; +} diff --git a/fdb-record-layer-core/src/main/proto/record_metadata.proto b/fdb-record-layer-core/src/main/proto/record_metadata.proto index bbf434c495..986aa80d87 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata.proto @@ -21,7 +21,8 @@ syntax = "proto2"; package com.apple.foundationdb.record; import "google/protobuf/descriptor.proto"; -import "record_value.proto"; +import "record_key_expression.proto"; +import "record_query_plan.proto"; option java_outer_classname = "RecordMetaDataProto"; @@ -72,7 +73,7 @@ message RecordType { required string name = 1; optional KeyExpression primary_key = 2; optional int32 since_version = 3; - optional com.apple.foundationdb.record.planprotos.Value explicit_key = 4; + optional Value explicit_key = 4; extensions 1000 to 2000; } @@ -103,94 +104,15 @@ message MetaData { } message UDF { - optional string name = 1; - optional com.apple.foundationdb.record.planprotos.PValue function_value = 3; -} - -message Then { - repeated KeyExpression child = 1; -} - -message List { - repeated KeyExpression child = 1; -} - -message Field { - required string field_name = 1; - required FanType fan_type = 2; - enum FanType { - SCALAR = 1; - FAN_OUT = 2; - CONCATENATE = 3; - } - optional NullInterpretation nullInterpretation = 3 [default = NOT_UNIQUE]; - enum NullInterpretation { - NOT_UNIQUE = 1; // Missing values allowed multiple times in unique index - UNIQUE = 2; // Missing values treated like null value in unique index - NOT_NULL = 3; // Missing values are the default for the type, not null - } -} - -message Nesting { - optional Field parent = 1; - optional KeyExpression child = 2; -} - -message Grouping { - required KeyExpression whole_key = 1; - optional int32 grouped_count = 3 [default = 1]; -} - -message Dimensions { - required KeyExpression whole_key = 1; - optional int32 prefix_size = 2; - optional int32 dimensions_size = 3; -} - -message KeyWithValue { - required KeyExpression inner_key = 1; - optional int32 split_point = 2 [default = 1]; -} - -message Split { - required KeyExpression joined = 1; - optional int32 split_size = 2; -} - -message Empty { + optional string function_name = 1; + optional com.apple.foundationdb.record.planprotos.PValue function_value = 2; } -message Version { -} - -message RecordTypeKey { -} -message Function { - required string name = 1; - required KeyExpression arguments = 2; -} - -message KeyExpression { - // Exactly one of the following: - optional Then then = 1; - optional Nesting nesting = 2; - optional Field field = 3; - optional Grouping grouping = 4; - optional Empty empty = 5; - optional Split split = 6; - optional Version version = 7; - optional com.apple.foundationdb.record.planprotos.Value value = 8; - optional Function function = 9; - optional KeyWithValue key_with_value = 10; - optional RecordTypeKey record_type_key = 11; - optional List list = 12; - optional Dimensions dimensions = 13; -} message JoinedRecordType { optional string name = 1; - optional com.apple.foundationdb.record.planprotos.Value record_type_key = 4; + optional Value record_type_key = 4; message JoinConstituent { optional string name = 1; @@ -211,7 +133,7 @@ message JoinedRecordType { message UnnestedRecordType { optional string name = 1; - optional com.apple.foundationdb.record.planprotos.Value record_type_key = 2; + optional Value record_type_key = 2; message NestedConstituent { optional string name = 1; @@ -239,7 +161,7 @@ enum ComparisonType { message SimpleComparison { optional ComparisonType type = 1; - optional com.apple.foundationdb.record.planprotos.Value operand = 2; + optional Value operand = 2; } // in Comparisons.Comparison we have a special NullComparison that takes care of comparing diff --git a/fdb-record-layer-core/src/main/proto/record_metadata_options.proto b/fdb-record-layer-core/src/main/proto/record_metadata_options.proto index 0a8e108786..c4426b9ef1 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata_options.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata_options.proto @@ -23,8 +23,8 @@ package com.apple.foundationdb.record; option java_outer_classname = "RecordMetaDataOptionsProto"; import "google/protobuf/descriptor.proto"; +import "record_key_expression.proto"; import "record_metadata.proto"; -import "record_value.proto"; message SchemaOptions { optional bool split_long_records = 3; @@ -44,7 +44,7 @@ message RecordTypeOptions { } optional Usage usage = 1 [default = UNSET]; optional int32 since_version = 2; // This record_type is introduced on this metadata version - optional com.apple.foundationdb.record.planprotos.Value record_type_key = 3; + optional Value record_type_key = 3; } extend google.protobuf.MessageOptions { diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 3e1725e08e..ed970a78ab 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -22,17 +22,933 @@ syntax = "proto2"; package com.apple.foundationdb.record.planprotos; import "google/protobuf/descriptor.proto"; import "google/protobuf/any.proto"; -import "record_metadata.proto"; -import "record_value.proto"; +import "record_key_expression.proto"; option java_outer_classname = "RecordQueryPlanProto"; option java_multiple_files = true; +// +// Type system +// +// Note that the subtypes are nested messages and we don't define an extension point because PType should be considered +// a sealed treat that cannot be extended. +// +message PType { + enum PTypeCode { + UNKNOWN = 1; + ANY = 2; + NULL = 3; + BOOLEAN = 4; + BYTES = 5; + DOUBLE = 6; + FLOAT = 7; + INT = 8; + LONG = 9; + STRING = 10; + VERSION = 11; + ENUM = 12; + RECORD = 13; + ARRAY = 14; + RELATION = 15; + NONE = 16; + } + + message PPrimitiveType { + optional PTypeCode type_code = 1; + optional bool is_nullable = 2; + } + + message PNullType { + // nothing + } + + message PNoneType { + // nothing + } + + message PAnyType { + // nothing + } + + message PAnyRecordType { + optional bool is_nullable = 1; + } + + message PEnumType { + message PEnumValue { + optional string name = 1; + optional int32 number = 2; + } + + optional bool is_nullable = 1; + repeated PEnumValue enum_values = 2; + optional string name = 3; // referential name -- not used in the planner + } + + message PRecordType { + message PField { + optional PType field_type = 1; + optional string field_name = 2; + optional int32 field_index = 3; + } + + optional int32 reference_id = 1; + optional string name = 2; // referential name -- not used in the planner + optional bool is_nullable = 3; + repeated PField fields = 4; + } + + message PRelationType { + optional PType inner_type = 1; + } + + message PArrayType { + optional bool is_nullable = 2; + optional PType element_type = 3; + } + + oneof specific_type { + PPrimitiveType primitive_type = 1; + PNullType null_type = 2; + PNoneType none_type = 3; + PAnyType any_type = 4; + PEnumType enum_type = 5; + PRecordType record_type = 6; + PRelationType relation_type = 7; + PArrayType array_type = 8; + PAnyRecordType any_record_type = 9; + } +} + +// +// Coercion Tries +// +message PCoercionTrieNode { + message IntChildPair { + optional int32 index = 1; + optional PCoercionTrieNode child_coercion_trie_node = 2; + } + optional bool children_map_is_null = 1; + repeated IntChildPair child_pair = 2; + optional PCoercionBiFunction value = 3; +} + +message PCoercionBiFunction { + extensions 5000 to max; + oneof specific_function { + google.protobuf.Any additional_coercion_bi_functions = 1; + PPrimitiveCoercionBiFunction primitive_coercion_bi_function = 2; + PArrayCoercionBiFunction array_coercion_bi_function = 3; + } +} + +message PPrimitiveCoercionBiFunction { + enum PPhysicalOperator { + INT_TO_LONG = 1; + INT_TO_FLOAT = 2; + INT_TO_DOUBLE = 3; + LONG_TO_FLOAT = 4; + LONG_TO_DOUBLE = 5; + FLOAT_TO_DOUBLE = 6; + NULL_TO_INT = 7; + NULL_TO_LONG = 8; + NULL_TO_FLOAT = 9; + NULL_TO_DOUBLE = 10; + NULL_TO_BOOLEAN = 11; + NULL_TO_STRING = 12; + NULL_TO_ARRAY = 13; + NULL_TO_RECORD = 14; + NONE_TO_ARRAY = 15; + NULL_TO_ENUM = 16; + STRING_TO_ENUM = 17; + } + optional PPhysicalOperator operator = 1; +} + +message PArrayCoercionBiFunction { + optional PType from_array_type = 1; + optional PType to_array_type = 2; + optional PCoercionTrieNode elements_trie = 3; +} + +message PTransformationTrieNode { + message IntChildPair { + optional int32 index = 1; + optional PTransformationTrieNode child_transformation_trie_node = 2; + } + optional bool children_map_is_null = 1; + repeated IntChildPair child_pair = 2; + optional PValue value = 3; +} + +message PFieldAccessTrieNode { + message PResolvedAccessorChildPair { + optional PFieldPath.PResolvedAccessor resolved_accessor = 1; + optional PFieldAccessTrieNode child_field_access_trie_node = 2; + } + optional bool children_map_is_null = 1; + repeated PResolvedAccessorChildPair child_pair = 2; + optional PType type = 3; +} + +// +// Values +// +message PValue { + extensions 5000 to max; + oneof specific_value { + google.protobuf.Any additional_values = 1; + PLightArrayConstructorValue light_array_constructor_value = 2; + PAndOrValue and_or_value = 3; + PArithmeticValue arithmetic_value = 4; + PConditionSelectorValue condition_selector_value = 5; + PConstantObjectValue constant_object_value = 6; + PConstantValue constant_value = 7; + PCountValue count_value = 8; + PDerivedValue derived_value = 9; + PEmptyValue empty_value = 10; + PExistsValue exists_value = 11; + PFieldValue field_value = 12; + PIndexedValue indexed_value = 13; + PMaxEverValue max_ever_value = 14; + PMinEverValue min_ever_value = 15; + PInOpValue in_op_value = 16; + PLikeOperatorValue like_operator_value = 17; + PLiteralValue literal_value = 18; + PNotValue not_value = 19; + PNullValue null_value = 20; + PNumericAggregationValue.PSum numeric_aggregation_value_sum = 21; + PNumericAggregationValue.PAvg numeric_aggregation_value_avg = 22; + PNumericAggregationValue.PMin numeric_aggregation_value_min = 23; + PNumericAggregationValue.PMax numeric_aggregation_value_max = 24; + PObjectValue object_value = 25; + POfTypeValue of_type_value = 26; + PPatternForLikeValue pattern_for_like_value = 27; + PPickValue pick_value = 28; + PPromoteValue promote_value = 29; + PQuantifiedObjectValue quantified_object_value = 30; + PQueriedValue queried_value = 31; + PRankValue rank_value = 32; + PRecordConstructorValue record_constructor_value = 33; + PRecordTypeValue record_type_value = 34; + PBinaryRelOpValue binary_rel_op_value = 35; + PUnaryRelOpValue unary_rel_op_value = 36; + PVariadicFunctionValue variadic_function_value = 37; + PVersionValue version_value = 38; + PFirstOrDefaultValue first_or_default_value = 39; + PThrowsValue throws_value = 40; + PIndexEntryObjectValue index_entry_object_value = 41; + PToOrderedBytesValue to_ordered_bytes_value = 42; + PFromOrderedBytesValue from_ordered_bytes_value = 43; + PCollateValue collate_value = 44; + PNumericAggregationValue.PBitmapConstructAgg numeric_aggregation_value_bitmap_construct_agg = 45; + PMacroFunctionValue macro_function_value = 46; + } +} + +message PMacroFunctionValue { + repeated PValue argument_list = 1; + optional PValue body = 2; +} + +message PAbstractArrayConstructorValue { + repeated PValue children = 1; + optional PType element_type = 2; +} + +message PLightArrayConstructorValue { + optional PAbstractArrayConstructorValue super = 1; +} + +message PAndOrValue { + enum POperator { + AND = 1; + OR = 2; + } + optional string function_name = 1; + optional PValue left_child = 2; + optional PValue right_child = 3; + optional POperator operator = 4; +} + +message PArithmeticValue { + enum PPhysicalOperator { + ADD_II = 1; + ADD_IL = 2; + ADD_IF = 3; + ADD_ID = 4; + ADD_IS = 5; + ADD_LI = 6; + ADD_LL = 7; + ADD_LF = 8; + ADD_LD = 9; + ADD_LS = 10; + ADD_FI = 11; + ADD_FL = 12; + ADD_FF = 13; + ADD_FD = 14; + ADD_FS = 15; + ADD_DI = 16; + ADD_DL = 17; + ADD_DF = 18; + ADD_DD = 19; + ADD_DS = 20; + ADD_SI = 21; + ADD_SL = 22; + ADD_SF = 23; + ADD_SD = 24; + ADD_SS = 25; + + SUB_II = 26; + SUB_IL = 27; + SUB_IF = 28; + SUB_ID = 29; + SUB_LI = 30; + SUB_LL = 31; + SUB_LF = 32; + SUB_LD = 33; + SUB_FI = 34; + SUB_FL = 35; + SUB_FF = 36; + SUB_FD = 37; + SUB_DI = 38; + SUB_DL = 39; + SUB_DF = 40; + SUB_DD = 41; + + MUL_II = 42; // my favorite + MUL_IL = 43; + MUL_IF = 44; + MUL_ID = 45; + MUL_LI = 46; + MUL_LL = 47; + MUL_LF = 48; + MUL_LD = 49; + MUL_FI = 50; + MUL_FL = 51; + MUL_FF = 52; + MUL_FD = 53; + MUL_DI = 54; + MUL_DL = 55; + MUL_DF = 56; + MUL_DD = 57; + + DIV_II = 58; + DIV_IL = 59; + DIV_IF = 60; + DIV_ID = 61; + DIV_LI = 62; + DIV_LL = 63; + DIV_LF = 64; + DIV_LD = 65; + DIV_FI = 66; + DIV_FL = 67; + DIV_FF = 68; + DIV_FD = 69; + DIV_DI = 70; + DIV_DL = 71; + DIV_DF = 72; + DIV_DD = 73; + + MOD_II = 74; + MOD_IL = 75; + MOD_IF = 76; + MOD_ID = 77; + MOD_LI = 78; + MOD_LL = 79; + MOD_LF = 80; + MOD_LD = 81; + MOD_FI = 82; + MOD_FL = 83; + MOD_FF = 84; + MOD_FD = 85; + MOD_DI = 86; + MOD_DL = 87; + MOD_DF = 88; + MOD_DD = 89; + + BITOR_II = 90; + BITOR_IL = 91; + BITOR_LI = 92; + BITOR_LL = 93; + + BITAND_II = 94; + BITAND_IL = 95; + BITAND_LI = 96; + BITAND_LL = 97; + + BITXOR_II = 98; + BITXOR_IL = 99; + BITXOR_LI = 100; + BITXOR_LL = 101; + + BITMAP_BUCKET_OFFSET_LI = 102; + BITMAP_BUCKET_OFFSET_II = 103; + BITMAP_BUCKET_NUMBER_LI = 104; + BITMAP_BUCKET_NUMBER_II = 105; + BITMAP_BIT_POSITION_LI = 106; + BITMAP_BIT_POSITION_II = 107; + } + + optional PPhysicalOperator operator = 1; + optional PValue left_child = 2; + optional PValue right_child = 3; +} + +message PConditionSelectorValue { + repeated PValue implications = 1; +} + +message PConstantObjectValue { + optional string alias = 1; + optional string constant_id= 2; + optional PType result_type = 3; +} + +message PConstantValue { + optional PValue value = 1; +} + +message PCountValue { + enum PPhysicalOperator { + COUNT = 1; + COUNT_STAR = 2; + } + optional PPhysicalOperator operator = 1; + optional PValue child = 2; +} + +message PDerivedValue { + repeated PValue children = 1; + optional PType result_type = 2; +} + +message PEmptyValue { + // nothing +} + +message PExistsValue { + optional PQuantifiedObjectValue child = 1; // deprecated + optional string alias = 2; +} + +message PFieldValue { + optional PValue child_value = 1; + optional PFieldPath field_path = 2; +} + +message PFieldPath { + message PResolvedAccessor { + optional string name = 1; + optional int32 ordinal = 2; + optional PType type = 3; + } + repeated PResolvedAccessor field_accessors = 1; +} + +message PFirstOrDefaultValue { + optional PValue child_value = 1; + optional PValue on_empty_result_value = 2; +} + +message PFromOrderedBytesValue { + optional PValue child = 1; + optional PDirection direction = 2; + optional PType result_type = 3; +} + +enum PDirection { + ASC_NULLS_FIRST = 1; + ASC_NULLS_LAST = 2; + DESC_NULLS_FIRST = 3; + DESC_NULLS_LAST = 4; +} + +message PIndexedValue { + optional PType result_type = 1; +} + +message PIndexEntryObjectValue { + optional string index_entry_alias = 1; + optional PIndexKeyValueToPartialRecord.PTupleSource source = 2; + repeated int32 ordinal_path = 3; + optional PType result_type = 4; +} + +message PIndexOnlyAggregateValue { // abstract + enum PPhysicalOperator { + MAX_EVER_LONG = 1; + MIN_EVER_LONG = 2; + } + optional PPhysicalOperator operator = 1; + optional PValue child = 2; +} + +message PInOpValue { + optional PValue probe_value = 1; + optional PValue in_array_value = 2; +} + +message PMaxEverValue { + optional PIndexOnlyAggregateValue super = 1; +} + +message PMinEverValue { + optional PIndexOnlyAggregateValue super = 1; +} + +message PLikeOperatorValue { + optional PValue src_child = 1; + optional PValue pattern_child = 2; +} + +message PLiteralValue { + optional PType result_type = 1; + optional PComparableObject value = 2; +} + +message PNotValue { + optional PValue child = 1; +} + +message PNullValue { + optional PType result_type = 1; +} + +message PNumericAggregationValue { + enum PPhysicalOperator { + SUM_I = 1; + SUM_L = 2; + SUM_F = 3; + SUM_D = 4; + + AVG_I = 5; + AVG_L = 6; + AVG_F = 7; + AVG_D = 8; + + MIN_I = 9; + MIN_L = 10; + MIN_F = 11; + MIN_D = 12; + + MAX_I = 13; + MAX_L = 14; + MAX_F = 15; + MAX_D = 16; + + BITMAP_CONSTRUCT_AGG_L = 17; + BITMAP_CONSTRUCT_AGG_I = 18; + } + optional PPhysicalOperator operator = 1; + optional PValue child = 2; + + message PBitmapConstructAgg { + optional PNumericAggregationValue super = 1; + } + message PSum { + optional PNumericAggregationValue super = 1; + } + message PAvg { + optional PNumericAggregationValue super = 1; + } + message PMin { + optional PNumericAggregationValue super = 1; + } + message PMax { + optional PNumericAggregationValue super = 1; + } +} + +message PObjectValue { + optional string alias = 1; + optional PType result_type = 2; +} + +message POfTypeValue { + optional PValue child = 1; + optional PType expected_type = 2; +} + +message PPatternForLikeValue { + optional PValue pattern_child = 1; + optional PValue escape_child = 2; +} + +message PPickValue { + optional PValue selector_value = 1; + repeated PValue alternative_values = 2; + optional PType result_type = 3; +} + +message PPromoteValue { + optional PValue in_value = 1; + optional PType promote_to_type = 2; + optional PCoercionTrieNode promotion_trie = 3; +} + +message PQuantifiedObjectValue { + optional string alias = 1; + optional PType result_type = 2; +} + +message PQueriedValue { + optional PType result_type = 1; + optional bool has_record_type_names = 3; + repeated string record_type_names = 2; +} + +message PRankValue { + optional PWindowedValue super = 1; +} + +message PRecordConstructorValue { + message PColumn { + optional PType.PRecordType.PField field = 1; + optional PValue value = 2; + } + optional PType result_type = 1; + repeated PColumn columns = 2; +} + +message PRecordTypeValue { + optional string alias = 1; // deprecated + optional PValue in = 2; +} + +message PRelOpValue { + optional string function_name = 1; + optional PComparison.PComparisonType comparison_type = 2; + repeated PValue children = 3; +} + +message PBinaryRelOpValue { + enum PBinaryPhysicalOperator { + EQ_BU = 1; + EQ_BB = 2; + EQ_IU = 3; + EQ_II = 4; + EQ_IL = 5; + EQ_IF = 6; + EQ_ID = 7; + EQ_LU = 8; + EQ_LI = 9; + EQ_LL = 10; + EQ_LF = 11; + EQ_LD = 12; + EQ_FU = 13; + EQ_FI = 14; + EQ_FL = 15; + EQ_FF = 16; + EQ_FD = 17; + EQ_DU = 18; + EQ_DI = 19; + EQ_DL = 20; + EQ_DF = 21; + EQ_DD = 22; + EQ_SU = 23; + EQ_SS = 24; + EQ_UU = 25; + EQ_UB = 26; + EQ_UI = 27; + EQ_UL = 28; + EQ_UF = 29; + EQ_UD = 30; + EQ_US = 31; + EQ_UV = 32; + EQ_VU = 33; + EQ_VV = 34; + NEQ_BU = 35; + NEQ_BB = 36; + NEQ_IU = 37; + NEQ_II = 38; + NEQ_IL = 39; + NEQ_IF = 40; + NEQ_ID = 41; + NEQ_LU = 42; + NEQ_LI = 43; + NEQ_LL = 44; + NEQ_LF = 45; + NEQ_LD = 46; + NEQ_FU = 47; + NEQ_FI = 48; + NEQ_FL = 49; + NEQ_FF = 50; + NEQ_FD = 51; + NEQ_DU = 52; + NEQ_DI = 53; + NEQ_DL = 54; + NEQ_DF = 55; + NEQ_DD = 56; + NEQ_SU = 57; + NEQ_SS = 58; + NEQ_UU = 59; + NEQ_UB = 60; + NEQ_UI = 61; + NEQ_UL = 62; + NEQ_UF = 63; + NEQ_UD = 64; + NEQ_US = 65; + NEQ_UV = 66; + NEQ_VU = 67; + NEQ_VV = 68; + LT_IU = 69; + LT_II = 70; + LT_IL = 71; + LT_IF = 72; + LT_ID = 73; + LT_LU = 74; + LT_LI = 75; + LT_LL = 76; + LT_LF = 77; + LT_LD = 78; + LT_FU = 79; + LT_FI = 80; + LT_FL = 81; + LT_FF = 82; + LT_FD = 83; + LT_DU = 84; + LT_DI = 85; + LT_DL = 86; + LT_DF = 87; + LT_DD = 88; + LT_SU = 89; + LT_SS = 90; + LT_UU = 91; + LT_UB = 92; + LT_UI = 93; + LT_UL = 94; + LT_UF = 95; + LT_UD = 96; + LT_US = 97; + LT_UV = 98; + LT_VU = 99; + LT_VV = 100; + LTE_IU = 101; + LTE_II = 102; + LTE_IL = 103; + LTE_IF = 104; + LTE_ID = 105; + LTE_LU = 106; + LTE_LI = 107; + LTE_LL = 108; + LTE_LF = 109; + LTE_LD = 110; + LTE_FU = 111; + LTE_FI = 112; + LTE_FL = 113; + LTE_FF = 114; + LTE_FD = 115; + LTE_DU = 116; + LTE_DI = 117; + LTE_DL = 118; + LTE_DF = 119; + LTE_DD = 120; + LTE_SU = 121; + LTE_SS = 122; + LTE_UU = 123; + LTE_UB = 124; + LTE_UI = 125; + LTE_UL = 126; + LTE_UF = 127; + LTE_UD = 128; + LTE_US = 129; + LTE_UV = 130; + LTE_VU = 131; + LTE_VV = 132; + GT_IU = 133; + GT_II = 134; + GT_IL = 135; + GT_IF = 136; + GT_ID = 137; + GT_LU = 138; + GT_LI = 139; + GT_LL = 140; + GT_LF = 141; + GT_LD = 142; + GT_FU = 143; + GT_FI = 144; + GT_FL = 145; + GT_FF = 146; + GT_FD = 147; + GT_DU = 148; + GT_DI = 149; + GT_DL = 150; + GT_DF = 151; + GT_DD = 152; + GT_SU = 153; + GT_SS = 154; + GT_UU = 155; + GT_UB = 156; + GT_UI = 157; + GT_UL = 158; + GT_UF = 159; + GT_UD = 160; + GT_US = 161; + GT_UV = 162; + GT_VU = 163; + GT_VV = 164; + GTE_IU = 165; + GTE_II = 166; + GTE_IL = 167; + GTE_IF = 168; + GTE_ID = 169; + GTE_LU = 170; + GTE_LI = 171; + GTE_LL = 172; + GTE_LF = 173; + GTE_LD = 174; + GTE_FU = 175; + GTE_FI = 176; + GTE_FL = 177; + GTE_FF = 178; + GTE_FD = 179; + GTE_DU = 180; + GTE_DI = 181; + GTE_DL = 182; + GTE_DF = 183; + GTE_DD = 184; + GTE_SU = 185; + GTE_SS = 186; + GTE_UU = 187; + GTE_UB = 188; + GTE_UI = 189; + GTE_UL = 190; + GTE_UF = 191; + GTE_UD = 192; + GTE_US = 193; + GTE_UV = 194; + GTE_VU = 195; + GTE_VV = 196; + + EQ_BYU = 197; + EQ_BYBY = 198; + EQ_UBY = 199; + NEQ_BYU = 200; + NEQ_BYBY = 201; + NEQ_UBY = 202; + LT_BYU = 203; + LT_BYBY = 204; + LTE_BYU = 205; + LTE_BYBY = 206; + GT_BYU = 207; + GT_BYBY = 208; + GTE_BYU = 209; + GTE_BYBY = 210; + + } + optional PRelOpValue super = 1; + optional PBinaryPhysicalOperator operator = 2; +} + +message PThrowsValue { + optional PType result_type = 1; +} + +message PToOrderedBytesValue { + optional PValue child = 1; + optional PDirection direction = 2; +} + +message PUnaryRelOpValue { + enum PUnaryPhysicalOperator { + IS_NULL_UI = 1; + IS_NULL_II = 2; + IS_NULL_LI = 3; + IS_NULL_FI = 4; + IS_NULL_DI = 5; + IS_NULL_SS = 6; + IS_NULL_BI = 7; + + IS_NOT_NULL_UI = 8; + IS_NOT_NULL_II = 9; + IS_NOT_NULL_LI = 10; + IS_NOT_NULL_FI = 11; + IS_NOT_NULL_DI = 12; + IS_NOT_NULL_SS = 13; + IS_NOT_NULL_BI = 14; + + IS_NULL_BY = 15; + IS_NOT_NULL_BY = 16; + } + optional PRelOpValue super = 1; + optional PUnaryPhysicalOperator operator = 2; +} + +message PUdfValue { // abstract + repeated PValue children = 1; + optional PType result_type = 2; +} + +message PVariadicFunctionValue { + enum PPhysicalOperator { + GREATEST_INT = 1; + GREATEST_LONG = 2; + GREATEST_BOOLEAN = 3; + GREATEST_STRING = 4; + GREATEST_FLOAT = 5; + GREATEST_DOUBLE = 6; + LEAST_INT = 7; + LEAST_LONG = 8; + LEAST_BOOLEAN = 9; + LEAST_STRING = 10; + LEAST_FLOAT = 11; + LEAST_DOUBLE = 12; + COALESCE_INT = 13; + COALESCE_LONG = 14; + COALESCE_BOOLEAN = 15; + COALESCE_STRING = 16; + COALESCE_FLOAT = 17; + COALESCE_DOUBLE = 18; + COALESCE_RECORD = 19; + COALESCE_ARRAY = 20; + } + optional PPhysicalOperator operator = 1; + repeated PValue children = 2; +} + +message PVersionValue { + optional string base_alias = 1; +} + +message PWindowedValue { + repeated PValue partitioning_values = 1; + repeated PValue argument_values = 2; +} + +message PCollateValue { + optional string collator_registry = 1; + optional PValue string_child = 2; + optional PValue locale_child = 3; + optional PValue strength_child = 4; +} // // Comparisons // message PComparison { + enum PComparisonType { + EQUALS = 1; + NOT_EQUALS = 2; + LESS_THAN = 3; + LESS_THAN_OR_EQUALS = 4; + GREATER_THAN = 5; + GREATER_THAN_OR_EQUALS = 6; + STARTS_WITH = 7; + NOT_NULL = 8; + IS_NULL = 9; + IN = 10; + TEXT_CONTAINS_ALL = 11; + TEXT_CONTAINS_ALL_WITHIN = 12; + TEXT_CONTAINS_ANY = 13; + TEXT_CONTAINS_PHRASE = 14; + TEXT_CONTAINS_PREFIX = 15; + TEXT_CONTAINS_ALL_PREFIXES = 16; + TEXT_CONTAINS_ANY_PREFIX = 17; + SORT = 18; + LIKE = 19; + } + extensions 5000 to max; oneof specific_comparison { google.protobuf.Any additional_comparisons = 1; @@ -51,12 +967,12 @@ message PComparison { } message PSimpleComparison { - optional PComparisonType type = 1; + optional PComparison.PComparisonType type = 1; optional PComparableObject object = 2; } message PConversionSimpleComparison { - optional PComparisonType type = 1; + optional PComparison.PComparisonType type = 1; optional PComparableObject object = 2; optional KeyExpression conversion = 3; } @@ -69,30 +985,30 @@ message PParameterComparison { CONSTANT = 4; } - optional PComparisonType type = 1; + optional PComparison.PComparisonType type = 1; optional string parameter = 2; optional PBindingKind internal = 3; } message PConversionParameterComparison { - optional PComparisonType type = 1; + optional PComparison.PComparisonType type = 1; optional string parameter = 2; optional PParameterComparison.PBindingKind internal = 3; optional KeyExpression conversion = 4; } message PValueComparison { - optional PComparisonType type = 1; + optional PComparison.PComparisonType type = 1; optional PValue comparandValue = 2; } message PListComparison { - optional PComparisonType type = 1; + optional PComparison.PComparisonType type = 1; repeated PComparableObject comparand = 2; } message PNullComparison { - optional PComparisonType type = 1; + optional PComparison.PComparisonType type = 1; } message POpaqueEqualityComparison { @@ -106,7 +1022,7 @@ message PMultiColumnComparison { message PInvertedFunctionComparison { optional Function function = 1; optional PComparison original_comparison = 2; - optional PComparisonType type = 3; + optional PComparison.PComparisonType type = 3; } message PRecordTypeComparison { @@ -320,6 +1236,12 @@ message PIndexKeyValueToPartialRecord { optional string invertible_function = 5; } + enum PTupleSource { + KEY = 1; + VALUE = 2; + OTHER = 3; + } + message PCopyIfPredicate { message PTruePredicate { // nothing @@ -811,3 +1733,27 @@ message PRecordQueryUnionPlanBase { message PRecordQueryUpdatePlan { optional PRecordQueryAbstractDataModificationPlan super = 1; } + +message PEnumLightValue { + optional string name = 1; + optional int32 number = 2; +} + +message PUUID { + optional uint64 mostSigBits = 1; + optional uint64 leastSigBits = 2; +} + +message PFDBRecordVersion { + optional bytes raw_bytes = 1; +} + +message PComparableObject { + oneof specific_object { + Value primitive_object = 1; + PEnumLightValue enum_object = 2; + PUUID uuid = 3; + PFDBRecordVersion fdb_record_version = 4; + bytes bytes_as_byte_string = 5; + } +} diff --git a/fdb-record-layer-core/src/main/proto/record_query_runtime.proto b/fdb-record-layer-core/src/main/proto/record_query_runtime.proto index 5d1cc7acee..221643ad50 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_runtime.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_runtime.proto @@ -21,7 +21,8 @@ syntax = "proto2"; package com.apple.foundationdb.record.planprotos; import "google/protobuf/descriptor.proto"; -import "record_value.proto"; +import "record_metadata.proto"; +import "record_query_plan.proto"; option java_outer_classname = "RecordQueryRuntimeProto"; option java_multiple_files = true; diff --git a/fdb-record-layer-core/src/main/proto/record_value.proto b/fdb-record-layer-core/src/main/proto/record_value.proto deleted file mode 100644 index 2bc87411f0..0000000000 --- a/fdb-record-layer-core/src/main/proto/record_value.proto +++ /dev/null @@ -1,970 +0,0 @@ -syntax = "proto2"; - -package com.apple.foundationdb.record.planprotos; -import "google/protobuf/descriptor.proto"; -import "google/protobuf/any.proto"; - -option java_outer_classname = "RecordMetaDataValueProto"; -option java_multiple_files = true; - -// -// Type system -// -// Note that the subtypes are nested messages and we don't define an extension point because PType should be considered -// a sealed treat that cannot be extended. -// -message PType { - enum PTypeCode { - UNKNOWN = 1; - ANY = 2; - NULL = 3; - BOOLEAN = 4; - BYTES = 5; - DOUBLE = 6; - FLOAT = 7; - INT = 8; - LONG = 9; - STRING = 10; - VERSION = 11; - ENUM = 12; - RECORD = 13; - ARRAY = 14; - RELATION = 15; - NONE = 16; - } - - message PPrimitiveType { - optional PTypeCode type_code = 1; - optional bool is_nullable = 2; - } - - message PNullType { - // nothing - } - - message PNoneType { - // nothing - } - - message PAnyType { - // nothing - } - - message PAnyRecordType { - optional bool is_nullable = 1; - } - - message PEnumType { - message PEnumValue { - optional string name = 1; - optional int32 number = 2; - } - - optional bool is_nullable = 1; - repeated PEnumValue enum_values = 2; - optional string name = 3; // referential name -- not used in the planner - } - - message PRecordType { - message PField { - optional PType field_type = 1; - optional string field_name = 2; - optional int32 field_index = 3; - } - - optional int32 reference_id = 1; - optional string name = 2; // referential name -- not used in the planner - optional bool is_nullable = 3; - repeated PField fields = 4; - } - - message PRelationType { - optional PType inner_type = 1; - } - - message PArrayType { - optional bool is_nullable = 2; - optional PType element_type = 3; - } - - oneof specific_type { - PPrimitiveType primitive_type = 1; - PNullType null_type = 2; - PNoneType none_type = 3; - PAnyType any_type = 4; - PEnumType enum_type = 5; - PRecordType record_type = 6; - PRelationType relation_type = 7; - PArrayType array_type = 8; - PAnyRecordType any_record_type = 9; - } -} - -// -// Coercion Tries -// -message PCoercionTrieNode { - message IntChildPair { - optional int32 index = 1; - optional PCoercionTrieNode child_coercion_trie_node = 2; - } - optional bool children_map_is_null = 1; - repeated IntChildPair child_pair = 2; - optional PCoercionBiFunction value = 3; -} - -message PCoercionBiFunction { - extensions 5000 to max; - oneof specific_function { - google.protobuf.Any additional_coercion_bi_functions = 1; - PPrimitiveCoercionBiFunction primitive_coercion_bi_function = 2; - PArrayCoercionBiFunction array_coercion_bi_function = 3; - } -} - -message PPrimitiveCoercionBiFunction { - enum PPhysicalOperator { - INT_TO_LONG = 1; - INT_TO_FLOAT = 2; - INT_TO_DOUBLE = 3; - LONG_TO_FLOAT = 4; - LONG_TO_DOUBLE = 5; - FLOAT_TO_DOUBLE = 6; - NULL_TO_INT = 7; - NULL_TO_LONG = 8; - NULL_TO_FLOAT = 9; - NULL_TO_DOUBLE = 10; - NULL_TO_BOOLEAN = 11; - NULL_TO_STRING = 12; - NULL_TO_ARRAY = 13; - NULL_TO_RECORD = 14; - NONE_TO_ARRAY = 15; - NULL_TO_ENUM = 16; - STRING_TO_ENUM = 17; - } - optional PPhysicalOperator operator = 1; -} - -message PArrayCoercionBiFunction { - optional PType from_array_type = 1; - optional PType to_array_type = 2; - optional PCoercionTrieNode elements_trie = 3; -} - -message PTransformationTrieNode { - message IntChildPair { - optional int32 index = 1; - optional PTransformationTrieNode child_transformation_trie_node = 2; - } - optional bool children_map_is_null = 1; - repeated IntChildPair child_pair = 2; - optional PValue value = 3; -} - -message PFieldAccessTrieNode { - message PResolvedAccessorChildPair { - optional PFieldPath.PResolvedAccessor resolved_accessor = 1; - optional PFieldAccessTrieNode child_field_access_trie_node = 2; - } - optional bool children_map_is_null = 1; - repeated PResolvedAccessorChildPair child_pair = 2; - optional PType type = 3; -} - - -// -// Values -// -message PValue { - extensions 5000 to max; - oneof specific_value { - google.protobuf.Any additional_values = 1; - PLightArrayConstructorValue light_array_constructor_value = 2; - PAndOrValue and_or_value = 3; - PArithmeticValue arithmetic_value = 4; - PConditionSelectorValue condition_selector_value = 5; - PConstantObjectValue constant_object_value = 6; - PConstantValue constant_value = 7; - PCountValue count_value = 8; - PDerivedValue derived_value = 9; - PEmptyValue empty_value = 10; - PExistsValue exists_value = 11; - PFieldValue field_value = 12; - PIndexedValue indexed_value = 13; - PMaxEverValue max_ever_value = 14; - PMinEverValue min_ever_value = 15; - PInOpValue in_op_value = 16; - PLikeOperatorValue like_operator_value = 17; - PLiteralValue literal_value = 18; - PNotValue not_value = 19; - PNullValue null_value = 20; - PNumericAggregationValue.PSum numeric_aggregation_value_sum = 21; - PNumericAggregationValue.PAvg numeric_aggregation_value_avg = 22; - PNumericAggregationValue.PMin numeric_aggregation_value_min = 23; - PNumericAggregationValue.PMax numeric_aggregation_value_max = 24; - PObjectValue object_value = 25; - POfTypeValue of_type_value = 26; - PPatternForLikeValue pattern_for_like_value = 27; - PPickValue pick_value = 28; - PPromoteValue promote_value = 29; - PQuantifiedObjectValue quantified_object_value = 30; - PQueriedValue queried_value = 31; - PRankValue rank_value = 32; - PRecordConstructorValue record_constructor_value = 33; - PRecordTypeValue record_type_value = 34; - PBinaryRelOpValue binary_rel_op_value = 35; - PUnaryRelOpValue unary_rel_op_value = 36; - PVariadicFunctionValue variadic_function_value = 37; - PVersionValue version_value = 38; - PFirstOrDefaultValue first_or_default_value = 39; - PThrowsValue throws_value = 40; - PIndexEntryObjectValue index_entry_object_value = 41; - PToOrderedBytesValue to_ordered_bytes_value = 42; - PFromOrderedBytesValue from_ordered_bytes_value = 43; - PCollateValue collate_value = 44; - PNumericAggregationValue.PBitmapConstructAgg numeric_aggregation_value_bitmap_construct_agg = 45; - PFunctionValue function_value = 46; - } -} - -message PAbstractArrayConstructorValue { - repeated PValue children = 1; - optional PType element_type = 2; -} - -message PLightArrayConstructorValue { - optional PAbstractArrayConstructorValue super = 1; -} - -message PAndOrValue { - enum POperator { - AND = 1; - OR = 2; - } - optional string function_name = 1; - optional PValue left_child = 2; - optional PValue right_child = 3; - optional POperator operator = 4; -} - -message PFunctionValue { - optional string function_name = 1; - repeated PValue children = 2; - optional PValue underlying = 3; - optional PType result_type = 4; -} - -message PArithmeticValue { - enum PPhysicalOperator { - ADD_II = 1; - ADD_IL = 2; - ADD_IF = 3; - ADD_ID = 4; - ADD_IS = 5; - ADD_LI = 6; - ADD_LL = 7; - ADD_LF = 8; - ADD_LD = 9; - ADD_LS = 10; - ADD_FI = 11; - ADD_FL = 12; - ADD_FF = 13; - ADD_FD = 14; - ADD_FS = 15; - ADD_DI = 16; - ADD_DL = 17; - ADD_DF = 18; - ADD_DD = 19; - ADD_DS = 20; - ADD_SI = 21; - ADD_SL = 22; - ADD_SF = 23; - ADD_SD = 24; - ADD_SS = 25; - - SUB_II = 26; - SUB_IL = 27; - SUB_IF = 28; - SUB_ID = 29; - SUB_LI = 30; - SUB_LL = 31; - SUB_LF = 32; - SUB_LD = 33; - SUB_FI = 34; - SUB_FL = 35; - SUB_FF = 36; - SUB_FD = 37; - SUB_DI = 38; - SUB_DL = 39; - SUB_DF = 40; - SUB_DD = 41; - - MUL_II = 42; // my favorite - MUL_IL = 43; - MUL_IF = 44; - MUL_ID = 45; - MUL_LI = 46; - MUL_LL = 47; - MUL_LF = 48; - MUL_LD = 49; - MUL_FI = 50; - MUL_FL = 51; - MUL_FF = 52; - MUL_FD = 53; - MUL_DI = 54; - MUL_DL = 55; - MUL_DF = 56; - MUL_DD = 57; - - DIV_II = 58; - DIV_IL = 59; - DIV_IF = 60; - DIV_ID = 61; - DIV_LI = 62; - DIV_LL = 63; - DIV_LF = 64; - DIV_LD = 65; - DIV_FI = 66; - DIV_FL = 67; - DIV_FF = 68; - DIV_FD = 69; - DIV_DI = 70; - DIV_DL = 71; - DIV_DF = 72; - DIV_DD = 73; - - MOD_II = 74; - MOD_IL = 75; - MOD_IF = 76; - MOD_ID = 77; - MOD_LI = 78; - MOD_LL = 79; - MOD_LF = 80; - MOD_LD = 81; - MOD_FI = 82; - MOD_FL = 83; - MOD_FF = 84; - MOD_FD = 85; - MOD_DI = 86; - MOD_DL = 87; - MOD_DF = 88; - MOD_DD = 89; - - BITOR_II = 90; - BITOR_IL = 91; - BITOR_LI = 92; - BITOR_LL = 93; - - BITAND_II = 94; - BITAND_IL = 95; - BITAND_LI = 96; - BITAND_LL = 97; - - BITXOR_II = 98; - BITXOR_IL = 99; - BITXOR_LI = 100; - BITXOR_LL = 101; - - BITMAP_BUCKET_OFFSET_LI = 102; - BITMAP_BUCKET_OFFSET_II = 103; - BITMAP_BUCKET_NUMBER_LI = 104; - BITMAP_BUCKET_NUMBER_II = 105; - BITMAP_BIT_POSITION_LI = 106; - BITMAP_BIT_POSITION_II = 107; - } - - optional PPhysicalOperator operator = 1; - optional PValue left_child = 2; - optional PValue right_child = 3; -} - -message PConditionSelectorValue { - repeated PValue implications = 1; -} - -message PConstantObjectValue { - optional string alias = 1; - optional string constant_id= 2; - optional PType result_type = 3; -} - -message PConstantValue { - optional PValue value = 1; -} - -message PCountValue { - enum PPhysicalOperator { - COUNT = 1; - COUNT_STAR = 2; - } - optional PPhysicalOperator operator = 1; - optional PValue child = 2; -} - -message PDerivedValue { - repeated PValue children = 1; - optional PType result_type = 2; -} - -message PEmptyValue { - // nothing -} - -message PExistsValue { - optional PQuantifiedObjectValue child = 1; // deprecated - optional string alias = 2; -} - -message PFieldValue { - optional PValue child_value = 1; - optional PFieldPath field_path = 2; -} - -message PFieldPath { - message PResolvedAccessor { - optional string name = 1; - optional int32 ordinal = 2; - optional PType type = 3; - } - repeated PResolvedAccessor field_accessors = 1; -} - -message PFirstOrDefaultValue { - optional PValue child_value = 1; - optional PValue on_empty_result_value = 2; -} - -message PFromOrderedBytesValue { - optional PValue child = 1; - optional PDirection direction = 2; - optional PType result_type = 3; -} - -enum PDirection { - ASC_NULLS_FIRST = 1; - ASC_NULLS_LAST = 2; - DESC_NULLS_FIRST = 3; - DESC_NULLS_LAST = 4; -} - -message PIndexedValue { - optional PType result_type = 1; -} - -message PIndexEntryObjectValue { - optional string index_entry_alias = 1; - optional PTupleSource source = 2; - repeated int32 ordinal_path = 3; - optional PType result_type = 4; -} - -message PIndexOnlyAggregateValue { // abstract - enum PPhysicalOperator { - MAX_EVER_LONG = 1; - MIN_EVER_LONG = 2; - } - optional PPhysicalOperator operator = 1; - optional PValue child = 2; -} - -message PInOpValue { - optional PValue probe_value = 1; - optional PValue in_array_value = 2; -} - -message PMaxEverValue { - optional PIndexOnlyAggregateValue super = 1; -} - -message PMinEverValue { - optional PIndexOnlyAggregateValue super = 1; -} - -message PLikeOperatorValue { - optional PValue src_child = 1; - optional PValue pattern_child = 2; -} - -message PLiteralValue { - optional PType result_type = 1; - optional PComparableObject value = 2; -} - -message PNotValue { - optional PValue child = 1; -} - -message PNullValue { - optional PType result_type = 1; -} - -message PNumericAggregationValue { - enum PPhysicalOperator { - SUM_I = 1; - SUM_L = 2; - SUM_F = 3; - SUM_D = 4; - - AVG_I = 5; - AVG_L = 6; - AVG_F = 7; - AVG_D = 8; - - MIN_I = 9; - MIN_L = 10; - MIN_F = 11; - MIN_D = 12; - - MAX_I = 13; - MAX_L = 14; - MAX_F = 15; - MAX_D = 16; - - BITMAP_CONSTRUCT_AGG_L = 17; - BITMAP_CONSTRUCT_AGG_I = 18; - } - optional PPhysicalOperator operator = 1; - optional PValue child = 2; - - message PBitmapConstructAgg { - optional PNumericAggregationValue super = 1; - } - message PSum { - optional PNumericAggregationValue super = 1; - } - message PAvg { - optional PNumericAggregationValue super = 1; - } - message PMin { - optional PNumericAggregationValue super = 1; - } - message PMax { - optional PNumericAggregationValue super = 1; - } -} - -message PObjectValue { - optional string alias = 1; - optional PType result_type = 2; -} - -message POfTypeValue { - optional PValue child = 1; - optional PType expected_type = 2; -} - -message PPatternForLikeValue { - optional PValue pattern_child = 1; - optional PValue escape_child = 2; -} - -message PPickValue { - optional PValue selector_value = 1; - repeated PValue alternative_values = 2; - optional PType result_type = 3; -} - -message PPromoteValue { - optional PValue in_value = 1; - optional PType promote_to_type = 2; - optional PCoercionTrieNode promotion_trie = 3; -} - -message PQuantifiedObjectValue { - optional string alias = 1; - optional PType result_type = 2; -} - -message PQueriedValue { - optional PType result_type = 1; - optional bool has_record_type_names = 3; - repeated string record_type_names = 2; -} - -message PRankValue { - optional PWindowedValue super = 1; -} - -message PRecordConstructorValue { - message PColumn { - optional PType.PRecordType.PField field = 1; - optional PValue value = 2; - } - optional PType result_type = 1; - repeated PColumn columns = 2; -} - -message PRecordTypeValue { - optional string alias = 1; // deprecated - optional PValue in = 2; -} - -message PRelOpValue { - optional string function_name = 1; - optional PComparisonType comparison_type = 2; - repeated PValue children = 3; -} - -message PBinaryRelOpValue { - enum PBinaryPhysicalOperator { - EQ_BU = 1; - EQ_BB = 2; - EQ_IU = 3; - EQ_II = 4; - EQ_IL = 5; - EQ_IF = 6; - EQ_ID = 7; - EQ_LU = 8; - EQ_LI = 9; - EQ_LL = 10; - EQ_LF = 11; - EQ_LD = 12; - EQ_FU = 13; - EQ_FI = 14; - EQ_FL = 15; - EQ_FF = 16; - EQ_FD = 17; - EQ_DU = 18; - EQ_DI = 19; - EQ_DL = 20; - EQ_DF = 21; - EQ_DD = 22; - EQ_SU = 23; - EQ_SS = 24; - EQ_UU = 25; - EQ_UB = 26; - EQ_UI = 27; - EQ_UL = 28; - EQ_UF = 29; - EQ_UD = 30; - EQ_US = 31; - EQ_UV = 32; - EQ_VU = 33; - EQ_VV = 34; - NEQ_BU = 35; - NEQ_BB = 36; - NEQ_IU = 37; - NEQ_II = 38; - NEQ_IL = 39; - NEQ_IF = 40; - NEQ_ID = 41; - NEQ_LU = 42; - NEQ_LI = 43; - NEQ_LL = 44; - NEQ_LF = 45; - NEQ_LD = 46; - NEQ_FU = 47; - NEQ_FI = 48; - NEQ_FL = 49; - NEQ_FF = 50; - NEQ_FD = 51; - NEQ_DU = 52; - NEQ_DI = 53; - NEQ_DL = 54; - NEQ_DF = 55; - NEQ_DD = 56; - NEQ_SU = 57; - NEQ_SS = 58; - NEQ_UU = 59; - NEQ_UB = 60; - NEQ_UI = 61; - NEQ_UL = 62; - NEQ_UF = 63; - NEQ_UD = 64; - NEQ_US = 65; - NEQ_UV = 66; - NEQ_VU = 67; - NEQ_VV = 68; - LT_IU = 69; - LT_II = 70; - LT_IL = 71; - LT_IF = 72; - LT_ID = 73; - LT_LU = 74; - LT_LI = 75; - LT_LL = 76; - LT_LF = 77; - LT_LD = 78; - LT_FU = 79; - LT_FI = 80; - LT_FL = 81; - LT_FF = 82; - LT_FD = 83; - LT_DU = 84; - LT_DI = 85; - LT_DL = 86; - LT_DF = 87; - LT_DD = 88; - LT_SU = 89; - LT_SS = 90; - LT_UU = 91; - LT_UB = 92; - LT_UI = 93; - LT_UL = 94; - LT_UF = 95; - LT_UD = 96; - LT_US = 97; - LT_UV = 98; - LT_VU = 99; - LT_VV = 100; - LTE_IU = 101; - LTE_II = 102; - LTE_IL = 103; - LTE_IF = 104; - LTE_ID = 105; - LTE_LU = 106; - LTE_LI = 107; - LTE_LL = 108; - LTE_LF = 109; - LTE_LD = 110; - LTE_FU = 111; - LTE_FI = 112; - LTE_FL = 113; - LTE_FF = 114; - LTE_FD = 115; - LTE_DU = 116; - LTE_DI = 117; - LTE_DL = 118; - LTE_DF = 119; - LTE_DD = 120; - LTE_SU = 121; - LTE_SS = 122; - LTE_UU = 123; - LTE_UB = 124; - LTE_UI = 125; - LTE_UL = 126; - LTE_UF = 127; - LTE_UD = 128; - LTE_US = 129; - LTE_UV = 130; - LTE_VU = 131; - LTE_VV = 132; - GT_IU = 133; - GT_II = 134; - GT_IL = 135; - GT_IF = 136; - GT_ID = 137; - GT_LU = 138; - GT_LI = 139; - GT_LL = 140; - GT_LF = 141; - GT_LD = 142; - GT_FU = 143; - GT_FI = 144; - GT_FL = 145; - GT_FF = 146; - GT_FD = 147; - GT_DU = 148; - GT_DI = 149; - GT_DL = 150; - GT_DF = 151; - GT_DD = 152; - GT_SU = 153; - GT_SS = 154; - GT_UU = 155; - GT_UB = 156; - GT_UI = 157; - GT_UL = 158; - GT_UF = 159; - GT_UD = 160; - GT_US = 161; - GT_UV = 162; - GT_VU = 163; - GT_VV = 164; - GTE_IU = 165; - GTE_II = 166; - GTE_IL = 167; - GTE_IF = 168; - GTE_ID = 169; - GTE_LU = 170; - GTE_LI = 171; - GTE_LL = 172; - GTE_LF = 173; - GTE_LD = 174; - GTE_FU = 175; - GTE_FI = 176; - GTE_FL = 177; - GTE_FF = 178; - GTE_FD = 179; - GTE_DU = 180; - GTE_DI = 181; - GTE_DL = 182; - GTE_DF = 183; - GTE_DD = 184; - GTE_SU = 185; - GTE_SS = 186; - GTE_UU = 187; - GTE_UB = 188; - GTE_UI = 189; - GTE_UL = 190; - GTE_UF = 191; - GTE_UD = 192; - GTE_US = 193; - GTE_UV = 194; - GTE_VU = 195; - GTE_VV = 196; - - EQ_BYU = 197; - EQ_BYBY = 198; - EQ_UBY = 199; - NEQ_BYU = 200; - NEQ_BYBY = 201; - NEQ_UBY = 202; - LT_BYU = 203; - LT_BYBY = 204; - LTE_BYU = 205; - LTE_BYBY = 206; - GT_BYU = 207; - GT_BYBY = 208; - GTE_BYU = 209; - GTE_BYBY = 210; - - } - optional PRelOpValue super = 1; - optional PBinaryPhysicalOperator operator = 2; -} - -message PThrowsValue { - optional PType result_type = 1; -} - -message PToOrderedBytesValue { - optional PValue child = 1; - optional PDirection direction = 2; -} - -message PUnaryRelOpValue { - enum PUnaryPhysicalOperator { - IS_NULL_UI = 1; - IS_NULL_II = 2; - IS_NULL_LI = 3; - IS_NULL_FI = 4; - IS_NULL_DI = 5; - IS_NULL_SS = 6; - IS_NULL_BI = 7; - - IS_NOT_NULL_UI = 8; - IS_NOT_NULL_II = 9; - IS_NOT_NULL_LI = 10; - IS_NOT_NULL_FI = 11; - IS_NOT_NULL_DI = 12; - IS_NOT_NULL_SS = 13; - IS_NOT_NULL_BI = 14; - - IS_NULL_BY = 15; - IS_NOT_NULL_BY = 16; - } - optional PRelOpValue super = 1; - optional PUnaryPhysicalOperator operator = 2; -} - -message PUdfValue { // abstract - repeated PValue children = 1; - optional PType result_type = 2; -} - -message PVariadicFunctionValue { - enum PPhysicalOperator { - GREATEST_INT = 1; - GREATEST_LONG = 2; - GREATEST_BOOLEAN = 3; - GREATEST_STRING = 4; - GREATEST_FLOAT = 5; - GREATEST_DOUBLE = 6; - LEAST_INT = 7; - LEAST_LONG = 8; - LEAST_BOOLEAN = 9; - LEAST_STRING = 10; - LEAST_FLOAT = 11; - LEAST_DOUBLE = 12; - COALESCE_INT = 13; - COALESCE_LONG = 14; - COALESCE_BOOLEAN = 15; - COALESCE_STRING = 16; - COALESCE_FLOAT = 17; - COALESCE_DOUBLE = 18; - COALESCE_RECORD = 19; - COALESCE_ARRAY = 20; - } - optional PPhysicalOperator operator = 1; - repeated PValue children = 2; -} - -message PVersionValue { - optional string base_alias = 1; -} - -message PWindowedValue { - repeated PValue partitioning_values = 1; - repeated PValue argument_values = 2; -} - -message PCollateValue { - optional string collator_registry = 1; - optional PValue string_child = 2; - optional PValue locale_child = 3; - optional PValue strength_child = 4; -} - -enum PComparisonType { - EQUALS = 1; - NOT_EQUALS = 2; - LESS_THAN = 3; - LESS_THAN_OR_EQUALS = 4; - GREATER_THAN = 5; - GREATER_THAN_OR_EQUALS = 6; - STARTS_WITH = 7; - NOT_NULL = 8; - IS_NULL = 9; - IN = 10; - TEXT_CONTAINS_ALL = 11; - TEXT_CONTAINS_ALL_WITHIN = 12; - TEXT_CONTAINS_ANY = 13; - TEXT_CONTAINS_PHRASE = 14; - TEXT_CONTAINS_PREFIX = 15; - TEXT_CONTAINS_ALL_PREFIXES = 16; - TEXT_CONTAINS_ANY_PREFIX = 17; - SORT = 18; - LIKE = 19; -} - -enum PTupleSource { - KEY = 1; - VALUE = 2; - OTHER = 3; -} - -message PEnumLightValue { - optional string name = 1; - optional int32 number = 2; -} - -message PUUID { - optional uint64 mostSigBits = 1; - optional uint64 leastSigBits = 2; -} - -message PFDBRecordVersion { - optional bytes raw_bytes = 1; -} - -message PComparableObject { - oneof specific_object { - Value primitive_object = 1; - PEnumLightValue enum_object = 2; - PUUID uuid = 3; - PFDBRecordVersion fdb_record_version = 4; - bytes bytes_as_byte_string = 5; - } -} - -message Value { - // TODO: This should be a oneof {} - optional double double_value = 1; - optional float float_value = 2; - optional int64 long_value = 3; - optional bool bool_value = 4; - optional string string_value = 5; - optional bytes bytes_value = 6; - optional int32 int_value = 7; -} diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidatorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidatorTest.java index d333c81f94..79f98a7d70 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidatorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidatorTest.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.async.RankedSet; import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataBuilder; import com.apple.foundationdb.record.RecordMetaDataOptionsProto; @@ -1020,7 +1021,7 @@ public void recordTypeKeyChanged() { RecordMetaDataProto.MetaData.Builder protoBuilder = metaData1.toProto().toBuilder() .setVersion(metaData1.getVersion() + 1); protoBuilder.getRecordTypesBuilder(0) - .setExplicitKey(com.apple.foundationdb.record.planprotos.Value.newBuilder() + .setExplicitKey(RecordKeyExpressionProto.Value.newBuilder() .setStringValue("new_key")); RecordMetaData metaData2 = RecordMetaData.build(protoBuilder.build()); assertInvalid("record type key changed", metaData1, metaData2); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataProtoTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataProtoTest.java index 7eb4d7678c..67f925abf1 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataProtoTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataProtoTest.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataBuilder; import com.apple.foundationdb.record.RecordMetaDataOptionsProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.TestRecords1Proto; import com.apple.foundationdb.record.TestRecords2Proto; @@ -270,10 +271,10 @@ public void indexProtoOptions() throws Exception { } } - public static RecordMetaDataProto.Field.Builder scalarField(String name) { - return RecordMetaDataProto.Field.newBuilder() + public static RecordKeyExpressionProto.Field.Builder scalarField(String name) { + return RecordKeyExpressionProto.Field.newBuilder() .setFieldName(name) - .setFanType(RecordMetaDataProto.Field.FanType.SCALAR); + .setFanType(RecordKeyExpressionProto.Field.FanType.SCALAR); } @Test @@ -287,23 +288,23 @@ public void versionstampIndexDeserialization() throws Exception { .setType(IndexTypes.VERSION) .addRecordType("MyRecord") .setRootExpression( - RecordMetaDataProto.KeyExpression.newBuilder() - .setNesting(RecordMetaDataProto.Nesting.newBuilder() + RecordKeyExpressionProto.KeyExpression.newBuilder() + .setNesting(RecordKeyExpressionProto.Nesting.newBuilder() .setParent(scalarField("header")) - .setChild(RecordMetaDataProto.KeyExpression.newBuilder() - .setThen(RecordMetaDataProto.Then.newBuilder() - .addChild(RecordMetaDataProto.KeyExpression.newBuilder() + .setChild(RecordKeyExpressionProto.KeyExpression.newBuilder() + .setThen(RecordKeyExpressionProto.Then.newBuilder() + .addChild(RecordKeyExpressionProto.KeyExpression.newBuilder() .setField(scalarField("num"))) - .addChild(RecordMetaDataProto.KeyExpression.newBuilder() - .setVersion(RecordMetaDataProto.Version.getDefaultInstance())))))); + .addChild(RecordKeyExpressionProto.KeyExpression.newBuilder() + .setVersion(RecordKeyExpressionProto.Version.getDefaultInstance())))))); protoBuilder.addRecordTypes(RecordMetaDataProto.RecordType.newBuilder() .setName("MyRecord") .setPrimaryKey( - RecordMetaDataProto.KeyExpression.newBuilder() - .setNesting(RecordMetaDataProto.Nesting.newBuilder() + RecordKeyExpressionProto.KeyExpression.newBuilder() + .setNesting(RecordKeyExpressionProto.Nesting.newBuilder() .setParent(scalarField("header")) - .setChild(RecordMetaDataProto.KeyExpression.newBuilder() + .setChild(RecordKeyExpressionProto.KeyExpression.newBuilder() .setField(scalarField("rec_no")))))); RecordMetaData metaData = RecordMetaData.newBuilder().addDependencies(BASE_DEPENDENCIES) @@ -326,25 +327,25 @@ public void indexGroupingCompatibility() throws Exception { .setName("RecordCount") .setType(IndexTypes.COUNT) .addRecordType("MyModernRecord") - .setRootExpression(RecordMetaDataProto.KeyExpression.newBuilder() - .setEmpty(RecordMetaDataProto.Empty.getDefaultInstance())); + .setRootExpression(RecordKeyExpressionProto.KeyExpression.newBuilder() + .setEmpty(RecordKeyExpressionProto.Empty.getDefaultInstance())); protoBuilder.addIndexesBuilder() .setName("MaxRecNo") .setType(IndexTypes.MAX_EVER) .addRecordType("MyModernRecord") - .setRootExpression(RecordMetaDataProto.KeyExpression.newBuilder() + .setRootExpression(RecordKeyExpressionProto.KeyExpression.newBuilder() .setField(scalarField("rec_no"))); protoBuilder.addIndexesBuilder() .setName("MaxRecNoGrouped") .setType(IndexTypes.MAX_EVER) .addRecordType("MyModernRecord") - .setRootExpression(RecordMetaDataProto.KeyExpression.newBuilder() - .setThen(RecordMetaDataProto.Then.newBuilder() - .addChild(RecordMetaDataProto.KeyExpression.newBuilder() + .setRootExpression(RecordKeyExpressionProto.KeyExpression.newBuilder() + .setThen(RecordKeyExpressionProto.Then.newBuilder() + .addChild(RecordKeyExpressionProto.KeyExpression.newBuilder() .setField(scalarField("index"))) - .addChild(RecordMetaDataProto.KeyExpression.newBuilder() + .addChild(RecordKeyExpressionProto.KeyExpression.newBuilder() .setField(scalarField("rec_no"))))); RecordMetaData metaData = RecordMetaData.newBuilder().addDependencies(BASE_DEPENDENCIES).setRecords(protoBuilder.build(), true).getRecordMetaData(); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java index 395cbca5b7..22649d0434 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.record.metadata; import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.expressions.BaseKeyExpression; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; @@ -87,7 +88,7 @@ public Message toProto() throws SerializationException { @Nonnull @Override - public RecordMetaDataProto.KeyExpression toKeyExpression() { + public RecordKeyExpressionProto.KeyExpression toKeyExpression() { throw new UnsupportedOperationException("UnknownKeyExpressions cannot be converted to Protobuf"); } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java index a59a35d24f..8a5dd491ee 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.record.metadata.expressions; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.common.text.TextSamples; @@ -69,7 +70,7 @@ public void serializationTest(@Nonnull Object value) throws InvalidProtocolBuffe final LiteralKeyExpression keyExpression = Key.Expressions.value(value); final LiteralKeyExpression parsedViaProto = LiteralKeyExpression.fromProto(keyExpression.toProto()); final LiteralKeyExpression parsedViaBytes = LiteralKeyExpression.fromProto( - com.apple.foundationdb.record.planprotos.Value.parseFrom(keyExpression.toProto().toByteArray())); + RecordKeyExpressionProto.Value.parseFrom(keyExpression.toProto().toByteArray())); assertEquals(keyExpression, parsedViaProto); assertEquals(keyExpression, parsedViaBytes); if (value instanceof byte[]) { @@ -90,7 +91,7 @@ public void incorrectUnicodeSurrogatePairSerializationTest() throws InvalidProto final LiteralKeyExpression parsedViaProto = LiteralKeyExpression.fromProto(keyExpression.toProto()); final LiteralKeyExpression parsedViaBytes = LiteralKeyExpression.fromProto( - com.apple.foundationdb.record.planprotos.Value.parseFrom(keyExpression.toProto().toByteArray())); + RecordKeyExpressionProto.Value.parseFrom(keyExpression.toProto().toByteArray())); assertEquals(keyExpression, parsedViaProto); // Comparison uses proto objects, so both sides have Longs. // Comparison with proto objects works since we never leave Java. diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java index 9c3e7c65af..b60f851094 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java @@ -26,6 +26,7 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataBuilder; import com.apple.foundationdb.record.RecordMetaDataOptionsProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.TestNoUnionEvolvedIllegalProto; import com.apple.foundationdb.record.TestNoUnionEvolvedProto; @@ -191,7 +192,7 @@ public void manyTypes() { metaData.addRecordTypesBuilder() .setName("type_" + ri) .getPrimaryKeyBuilder().getFieldBuilder() - .setFanType(RecordMetaDataProto.Field.FanType.SCALAR) + .setFanType(RecordKeyExpressionProto.Field.FanType.SCALAR) .setFieldName("field_1"); } @@ -218,10 +219,10 @@ public void historyCompat() { metaData.setRecords(TestRecords1Proto.getDescriptor().toProto()); metaData.addRecordTypesBuilder() .setName("MySimpleRecord") - .getPrimaryKeyBuilder().getFieldBuilder().setFieldName("rec_no").setFanType(RecordMetaDataProto.Field.FanType.SCALAR); + .getPrimaryKeyBuilder().getFieldBuilder().setFieldName("rec_no").setFanType(RecordKeyExpressionProto.Field.FanType.SCALAR); metaData.addRecordTypesBuilder() .setName("MyOtherRecord") - .getPrimaryKeyBuilder().getFieldBuilder().setFieldName("rec_no").setFanType(RecordMetaDataProto.Field.FanType.SCALAR); + .getPrimaryKeyBuilder().getFieldBuilder().setFieldName("rec_no").setFanType(RecordKeyExpressionProto.Field.FanType.SCALAR); metaData.setVersion(101); metaDataStore.saveRecordMetaData(metaData.build()); @@ -259,7 +260,7 @@ public void historyCompat() { metaData.setRecords(TestRecords1Proto.getDescriptor().toProto()); metaData.addRecordTypesBuilder() .setName("MySimpleRecord") - .getPrimaryKeyBuilder().getFieldBuilder().setFieldName("rec_no").setFanType(RecordMetaDataProto.Field.FanType.SCALAR); + .getPrimaryKeyBuilder().getFieldBuilder().setFieldName("rec_no").setFanType(RecordKeyExpressionProto.Field.FanType.SCALAR); metaData.addIndexesBuilder() .setName("MyIndex") .addRecordType("MySimpleRecord") @@ -267,10 +268,10 @@ public void historyCompat() { .setLastModifiedVersion(102) .getRootExpressionBuilder().getFieldBuilder() .setFieldName("num_value_2") - .setFanType(RecordMetaDataProto.Field.FanType.SCALAR); + .setFanType(RecordKeyExpressionProto.Field.FanType.SCALAR); metaData.addRecordTypesBuilder() .setName("MyOtherRecord") - .getPrimaryKeyBuilder().getFieldBuilder().setFieldName("rec_no").setFanType(RecordMetaDataProto.Field.FanType.SCALAR); + .getPrimaryKeyBuilder().getFieldBuilder().setFieldName("rec_no").setFanType(RecordKeyExpressionProto.Field.FanType.SCALAR); metaData.setVersion(102); metaDataStore.saveRecordMetaData(metaData.build()); context.commit(); diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTest.java index 855bf4eb5e..128fe3dbdc 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTest.java @@ -35,6 +35,7 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataBuilder; import com.apple.foundationdb.record.RecordMetaDataOptionsProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.RecordMetaDataProvider; import com.apple.foundationdb.record.ScanProperties; @@ -770,7 +771,7 @@ public void testAccessUserField() throws Exception { assertEquals("µs", recordStore.getHeaderUserField("foo").toStringUtf8()); ByteString bazValue = recordStore.getHeaderUserField("baz"); assertNotNull(bazValue); - KeyExpression expr = KeyExpression.fromProto(RecordMetaDataProto.KeyExpression.parseFrom(bazValue)); + KeyExpression expr = KeyExpression.fromProto(RecordKeyExpressionProto.KeyExpression.parseFrom(bazValue)); assertEquals(field("baz"), expr); // Add in a new field diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/storestate/FDBRecordStoreStateCacheTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/storestate/FDBRecordStoreStateCacheTest.java index 74c1586fe9..17a5a50429 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/storestate/FDBRecordStoreStateCacheTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/storestate/FDBRecordStoreStateCacheTest.java @@ -25,7 +25,7 @@ import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataBuilder; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.TestRecords1Proto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; @@ -790,7 +790,7 @@ public void cacheUserFields(@Nonnull StateCacheTestContext testContext) throws E try (FDBRecordContext context = testContext.getCachedContext(fdb, storeBuilder, FDBRecordStoreBase.StoreExistenceCheck.ERROR_IF_NOT_EXISTS)) { openSimpleRecordStore(context); assertNotNull(recordStore.getHeaderUserField("expr")); - KeyExpression expr = KeyExpression.fromProto(RecordMetaDataProto.KeyExpression.parseFrom(recordStore.getHeaderUserField("expr"))); + KeyExpression expr = KeyExpression.fromProto(RecordKeyExpressionProto.KeyExpression.parseFrom(recordStore.getHeaderUserField("expr"))); assertEquals(Key.Expressions.field("parent").nest("child"), expr); recordStore.clearHeaderUserField("expr"); commit(context); diff --git a/fdb-record-layer-lucene/src/main/proto/lucene_record_query_plan.proto b/fdb-record-layer-lucene/src/main/proto/lucene_record_query_plan.proto index d3e647b92b..76a1e307aa 100644 --- a/fdb-record-layer-lucene/src/main/proto/lucene_record_query_plan.proto +++ b/fdb-record-layer-lucene/src/main/proto/lucene_record_query_plan.proto @@ -21,7 +21,7 @@ syntax = "proto2"; package com.apple.foundationdb.record.planprotos; import "google/protobuf/descriptor.proto"; -import "record_metadata.proto"; +import "record_key_expression.proto"; import "record_query_plan.proto"; option java_outer_classname = "LuceneRecordQueryPlanProto"; From 0daa4ce7aed1655c1eb655647badcc5f6c60c41a Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 18 Dec 2024 14:35:23 -0800 Subject: [PATCH 11/41] checkstyle --- .../foundationdb/record/RecordMetaData.java | 17 +++++++----- .../record/RecordMetaDataBuilder.java | 12 ++++----- .../foundationdb/record/metadata/Key.java | 1 - .../record/metadata/{UDF.java => Udf.java} | 15 +++++++---- .../expressions/FieldKeyExpression.java | 1 - .../plan/cascades/NullableArrayTypeUtils.java | 1 - .../cascades/values/MacroFunctionValue.java | 26 +++++++++++-------- .../src/main/proto/record_metadata.proto | 4 +-- .../src/main/proto/record_query_plan.proto | 2 +- .../record/metadata/UnknownKeyExpression.java | 1 - .../expressions/LiteralKeyExpressionTest.java | 1 - 11 files changed, 44 insertions(+), 37 deletions(-) rename fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/{UDF.java => Udf.java} (86%) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java index 35e3b9efe7..f01fe23205 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java @@ -28,7 +28,7 @@ import com.apple.foundationdb.record.metadata.MetaDataException; import com.apple.foundationdb.record.metadata.RecordType; import com.apple.foundationdb.record.metadata.SyntheticRecordType; -import com.apple.foundationdb.record.metadata.UDF; +import com.apple.foundationdb.record.metadata.Udf; import com.apple.foundationdb.record.metadata.UnnestedRecordType; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression; @@ -47,7 +47,6 @@ import java.util.Comparator; import java.util.HashMap; import java.util.LinkedHashMap; -import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.TreeMap; @@ -85,7 +84,7 @@ public class RecordMetaData implements RecordMetaDataProvider { @Nonnull private final Map> recordTypeKeyToSyntheticTypeMap; @Nonnull - private final Map udfMap; + private final Map udfMap; @Nonnull private final Map indexes; @Nonnull @@ -136,7 +135,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor, @Nonnull Map indexes, @Nonnull Map universalIndexes, @Nonnull List formerIndexes, - @Nonnull Map udfMap, + @Nonnull Map udfMap, boolean splitLongRecords, boolean storeRecordVersions, int version, @@ -350,10 +349,14 @@ public List getFormerIndexes() { } @Nullable - public UDF getUDF(@Nonnull String name) {return udfMap.get(name);} + public Udf getUdf(@Nonnull String name) { + return udfMap.get(name); + } @Nonnull - public Collection getAllUDFs() {return udfMap.values();} + public Collection getAllUdfs() { + return udfMap.values(); + } public boolean isSplitLongRecords() { return splitLongRecords; @@ -706,7 +709,7 @@ public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor } // Add in the final options. - builder.addAllUdfs(udfMap.values().stream().map(UDF::toProto).collect(Collectors.toList())); + builder.addAllUdfs(udfMap.values().stream().map(Udf::toProto).collect(Collectors.toList())); builder.setSplitLongRecords(splitLongRecords); builder.setStoreRecordVersions(storeRecordVersions); builder.setVersion(version); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index d35d2e5acf..ee53fc2f9e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -35,7 +35,7 @@ import com.apple.foundationdb.record.metadata.RecordTypeIndexesBuilder; import com.apple.foundationdb.record.metadata.SyntheticRecordType; import com.apple.foundationdb.record.metadata.SyntheticRecordTypeBuilder; -import com.apple.foundationdb.record.metadata.UDF; +import com.apple.foundationdb.record.metadata.Udf; import com.apple.foundationdb.record.metadata.UnnestedRecordTypeBuilder; import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; @@ -114,7 +114,7 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider { @Nonnull private final Map> syntheticRecordTypes; @Nonnull - private final Map udfMap; + private final Map udfMap; @Nonnull private final Map indexes; @Nonnull @@ -231,8 +231,8 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa } PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, PlanHashable.CURRENT_FOR_CONTINUATION); - for (RecordMetaDataProto.UDF udf: metaDataProto.getUdfsList()) { - udfMap.put(udf.getFunctionName(), new UDF(udf.getFunctionName(), Value.fromValueProto(serializationContext, udf.getFunctionValue()))); + for (RecordMetaDataProto.Udf udf: metaDataProto.getUdfsList()) { + udfMap.put(udf.getFunctionName(), new Udf(udf.getFunctionName(), Value.fromValueProto(serializationContext, udf.getFunctionValue()))); } if (metaDataProto.hasSplitLongRecords()) { splitLongRecords = metaDataProto.getSplitLongRecords(); @@ -1191,11 +1191,11 @@ public void addFormerIndex(@Nonnull FormerIndex formerIndex) { formerIndexes.add(formerIndex); } - public void addUDF(@Nonnull UDF udf) { + public void addUdf(@Nonnull Udf udf) { udfMap.put(udf.getFunctionName(), udf); } - public void addUDFs(@Nonnull Collection udfs) { + public void addUdfs(@Nonnull Collection udfs) { udfs.forEach(udf -> udfMap.put(udf.getFunctionName(), udf)); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java index 8c9d75f29f..c930bdd688 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java @@ -23,7 +23,6 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; import com.apple.foundationdb.record.RecordKeyExpressionProto; -import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression; import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java similarity index 86% rename from fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java rename to fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java index ecd1b9cca7..417b5f9c3e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/UDF.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java @@ -28,12 +28,15 @@ import javax.annotation.Nonnull; -public class UDF { +/** + * Defines a User-defined-function + */ +public class Udf { @Nonnull private final String functionName; @Nonnull private final Value functionValue; - public UDF(@Nonnull String functionName, @Nonnull Value functionValue) { + public Udf(@Nonnull String functionName, @Nonnull Value functionValue) { this.functionName = functionName; this.functionValue = functionValue; } @@ -44,13 +47,15 @@ public String getFunctionName() { } @Nonnull - public Value getValue() {return functionValue;} + public Value getValue() { + return functionValue; + } @Nonnull - public RecordMetaDataProto.UDF toProto() { + public RecordMetaDataProto.Udf toProto() { PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, PlanHashable.CURRENT_FOR_CONTINUATION); - return RecordMetaDataProto.UDF.newBuilder() + return RecordMetaDataProto.Udf.newBuilder() .setFunctionName(functionName) .setFunctionValue(functionValue.toValueProto(serializationContext)) .build(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java index d47e31dcca..6cf17f350c 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java @@ -25,7 +25,6 @@ import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.RecordKeyExpressionProto; -import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/NullableArrayTypeUtils.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/NullableArrayTypeUtils.java index 35bd7d5bef..f741723aea 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/NullableArrayTypeUtils.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/NullableArrayTypeUtils.java @@ -21,7 +21,6 @@ package com.apple.foundationdb.record.query.plan.cascades; import com.apple.foundationdb.record.RecordKeyExpressionProto; -import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.expressions.NestingKeyExpression; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.MessageHelpers; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java index cd30bc3a3a..6432bf0eb3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java @@ -1,5 +1,5 @@ /* - * FunctionValue.java + * MacroFunctionValue.java * * This source file is part of the FoundationDB open source project * @@ -29,6 +29,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.query.plan.cascades.SemanticException; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; import com.google.auto.service.AutoService; import com.google.common.collect.ImmutableList; @@ -39,8 +40,14 @@ import java.util.List; import java.util.stream.Collectors; +/** + * The class defines a function + * The function takes in a list of arguments as QuantifiedObjectValue, of which alias = uniqueId, resultType = argument type + * and a body value, executing the function is by replacing the QuantifiedObjectValue in the body with the actual argument + */ public class MacroFunctionValue extends AbstractValue{ + @Nonnull private final List argList; @Nonnull @@ -74,21 +81,19 @@ public Object eval(@Nullable final FDBRecordStoreBase sto } @Nonnull - public static MacroFunctionValue of(@Nonnull final List argList, @Nonnull final Value underlying, @Nonnull final Type resultType) { - return new MacroFunctionValue(argList, underlying); + public static MacroFunctionValue of(@Nonnull final List argList, @Nonnull final Value body) { + return new MacroFunctionValue(argList, body); } @Nullable - public Value call(@Nonnull List arguments) { + public Value call(@Nonnull List arguments) { // replace the QuantifiedObjectValue in body with arguments SemanticException.check(arguments.size() == argList.size(), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument length doesn't match with function definition"); TranslationMap.Builder translationMapBuilder = new TranslationMap.Builder(); for (int i = 0; i < arguments.size(); i++) { - // check if arguments[i] type matches with argList[i] type + // check that arguments[i] type matches with argList[i] type final int finalI = i; - SemanticException.check(arguments.get(finalI) instanceof Value, SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "function input arguments must be of type Value"); - final var currentArg = (Value) arguments.get(finalI); - SemanticException.check(currentArg.getResultType().equals(argList.get(i).getResultType()), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument type doesn't match with function definition"); + SemanticException.check(arguments.get(finalI).getResultType().equals(argList.get(i).getResultType()), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument type doesn't match with function definition"); translationMapBuilder.when(((QuantifiedObjectValue) argList.get(i)).getAlias()).then((sourceAlias, leafValue) -> (Value)arguments.get(finalI)); } return body.translateCorrelations(translationMapBuilder.build()); @@ -103,7 +108,7 @@ public int hashCodeWithoutChildren() { @Override public PMacroFunctionValue toProto(@Nonnull final PlanSerializationContext serializationContext) { PMacroFunctionValue.Builder builder = PMacroFunctionValue.newBuilder(); - argList.forEach(arg -> builder.addArgumentList(arg.toValueProto(serializationContext))); + argList.forEach(arg -> builder.addArguments(arg.toValueProto(serializationContext))); return builder.setBody(body.toValueProto(serializationContext)).build(); } @@ -117,7 +122,7 @@ public PValue toValueProto(@Nonnull final PlanSerializationContext serialization @Nonnull public static MacroFunctionValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PMacroFunctionValue functionValue) { - return new MacroFunctionValue(functionValue.getArgumentListList().stream().map(pvalue -> Value.fromValueProto(serializationContext, pvalue)).collect(Collectors.toList()), Value.fromValueProto(serializationContext, functionValue.getBody())); + return new MacroFunctionValue(functionValue.getArgumentsList().stream().map(pvalue -> Value.fromValueProto(serializationContext, pvalue)).collect(Collectors.toList()), Value.fromValueProto(serializationContext, functionValue.getBody())); } @Nonnull @@ -151,4 +156,3 @@ public MacroFunctionValue fromProto(@Nonnull final PlanSerializationContext seri } } } - diff --git a/fdb-record-layer-core/src/main/proto/record_metadata.proto b/fdb-record-layer-core/src/main/proto/record_metadata.proto index 986aa80d87..832acb465e 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata.proto @@ -99,11 +99,11 @@ message MetaData { optional bool uses_subspace_key_counter = 11; repeated JoinedRecordType joined_record_types = 12; repeated UnnestedRecordType unnested_record_types = 13; - repeated UDF udfs = 14; + repeated Udf udfs = 14; extensions 1000 to 2000; } -message UDF { +message Udf { optional string function_name = 1; optional com.apple.foundationdb.record.planprotos.PValue function_value = 2; } diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index ed970a78ab..9d972fc552 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -247,7 +247,7 @@ message PValue { } message PMacroFunctionValue { - repeated PValue argument_list = 1; + repeated PValue arguments = 1; optional PValue body = 2; } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java index 22649d0434..04a884ad9e 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java @@ -22,7 +22,6 @@ import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.RecordKeyExpressionProto; -import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.expressions.BaseKeyExpression; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java index 8a5dd491ee..86e8815fb0 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java @@ -21,7 +21,6 @@ package com.apple.foundationdb.record.metadata.expressions; import com.apple.foundationdb.record.RecordKeyExpressionProto; -import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.common.text.TextSamples; import com.google.protobuf.InvalidProtocolBufferException; From 9a325bf0ec37911c473a605064910e9e1d60e171 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 18 Dec 2024 16:15:50 -0800 Subject: [PATCH 12/41] checksylte --- .../java/com/apple/foundationdb/record/metadata/Udf.java | 2 +- .../query/plan/cascades/values/MacroFunctionValue.java | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java index 417b5f9c3e..b7be2ddf44 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java @@ -29,7 +29,7 @@ import javax.annotation.Nonnull; /** - * Defines a User-defined-function + * Defines a User-defined-function. */ public class Udf { @Nonnull private final String functionName; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java index 6432bf0eb3..f178cbb118 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java @@ -41,9 +41,9 @@ import java.util.stream.Collectors; /** - * The class defines a function - * The function takes in a list of arguments as QuantifiedObjectValue, of which alias = uniqueId, resultType = argument type - * and a body value, executing the function is by replacing the QuantifiedObjectValue in the body with the actual argument + * The class defines a function. + * The function takes in a list of arguments as QuantifiedObjectValue, of which alias = uniqueId, resultType = argument type. + * and a body value, executing the function is by replacing the QuantifiedObjectValue in the body with the actual argument. */ public class MacroFunctionValue extends AbstractValue{ From 46eaaf1a4a328610702245efef86f64b8bfd91c6 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 19 Dec 2024 12:12:50 -0800 Subject: [PATCH 13/41] checkstyle --- .../record/query/plan/cascades/values/MacroFunctionValue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java index f178cbb118..b7c485c0b3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java @@ -45,7 +45,7 @@ * The function takes in a list of arguments as QuantifiedObjectValue, of which alias = uniqueId, resultType = argument type. * and a body value, executing the function is by replacing the QuantifiedObjectValue in the body with the actual argument. */ -public class MacroFunctionValue extends AbstractValue{ +public class MacroFunctionValue extends AbstractValue { @Nonnull private final List argList; From 523859e0707f04bdd22dcb23fc26c132075d3047 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 9 Jan 2025 12:14:46 -0800 Subject: [PATCH 14/41] save --- .../query/plan/cascades/BuiltInFunction.java | 88 +---------- .../record/query/plan/cascades/Function.java | 148 ++++++++++++++++++ 2 files changed, 153 insertions(+), 83 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java index 500f1a8978..dc15304c38 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java @@ -41,22 +41,7 @@ * @param The resulting type of the function. */ @SuppressWarnings("PMD.AbstractClassWithoutAbstractMethod") -public abstract class BuiltInFunction { - @Nonnull - final String functionName; - - @Nonnull - final List parameterTypes; - - /** - * The type of the function's variadic parameters (if any). - */ - @Nullable - final Type variadicSuffixType; - - @Nonnull - final EncapsulationFunction encapsulationFunction; - +public abstract class BuiltInFunction extends Function{ /** * Creates a new instance of {@link BuiltInFunction}. * @param functionName The name of the function. @@ -64,7 +49,7 @@ public abstract class BuiltInFunction { * @param encapsulationFunction An encapsulation of the function's runtime computation. */ public BuiltInFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final EncapsulationFunction encapsulationFunction) { - this(functionName, parameterTypes, null, encapsulationFunction); + super(functionName, parameterTypes, null, encapsulationFunction); } /** @@ -75,52 +60,7 @@ public BuiltInFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nullable final Type variadicSuffixType, @Nonnull final EncapsulationFunction encapsulationFunction) { - this.functionName = functionName; - this.parameterTypes = ImmutableList.copyOf(parameterTypes); - this.variadicSuffixType = variadicSuffixType; - this.encapsulationFunction = encapsulationFunction; - } - - @Nonnull - public String getFunctionName() { - return functionName; - } - - @Nonnull - public List getParameterTypes() { - return parameterTypes; - } - - @Nonnull - public Type resolveParameterType(int index) { - Verify.verify(index >= 0, "unexpected negative parameter index"); - if (index < parameterTypes.size()) { - return parameterTypes.get(index); - } else { - if (hasVariadicSuffix()) { - return variadicSuffixType; - } - throw new IllegalArgumentException("cannot resolve declared parameter at index " + index); - } - } - - @Nonnull - public List resolveParameterTypes(int numberOfArguments) { - Verify.verify(numberOfArguments > 0, "unexpected number of arguments"); - final ImmutableList.Builder resultBuilder = ImmutableList.builder(); - for (int i = 0; i < numberOfArguments; i ++) { - resultBuilder.add(resolveParameterType(i)); - } - return resultBuilder.build(); - } - - @Nullable - public Type getVariadicSuffixType() { - return variadicSuffixType; - } - - public boolean hasVariadicSuffix() { - return variadicSuffixType != null; + super(functionName, parameterTypes, variadicSuffixType, encapsulationFunction); } /** @@ -163,27 +103,9 @@ public Optional> validateCall(@Nonnull final List argum return Optional.of(this); } - @Nonnull - public EncapsulationFunction getEncapsulationFunction() { - return encapsulationFunction; - } - - @Nonnull - public Typed encapsulate(@Nonnull final List arguments) { - return encapsulationFunction.encapsulate(this, arguments); - } - @Nonnull @Override - public String toString() { - String variadicSuffixString = ""; - if (variadicSuffixType != null) { - variadicSuffixString = variadicSuffixType + "..."; - if (!parameterTypes.isEmpty()) { - variadicSuffixString = ", " + variadicSuffixString; - } - } - - return functionName + "(" + parameterTypes.stream().map(Object::toString).collect(Collectors.joining(",")) + variadicSuffixString + ")"; + public T encapsulate(@Nonnull final List arguments) { + return encapsulationFunction.encapsulate(this, arguments); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java new file mode 100644 index 0000000000..26efd4a92c --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java @@ -0,0 +1,148 @@ +/* + * BuiltInFunction.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades; + +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + +/** + * Main interface for defining a built-in function that can be evaluated against a number of arguments. + * + * A function could have a fixed number of arguments such as Value Add(Value, Value), or a + * variable number of arguments, such as Value TEXT_CONTAINS_ALL_PREFIXES(Value, Value, Value, Value, ...). + * + * @param The resulting type of the function. + */ +@SuppressWarnings("PMD.AbstractClassWithoutAbstractMethod") +public abstract class Function{ + @Nonnull + final String functionName; + + @Nonnull + final List parameterTypes; + + /** + * The type of the function's variadic parameters (if any). + */ + @Nullable + final Type variadicSuffixType; + + @Nonnull + final EncapsulationFunction encapsulationFunction; + + /** + * Creates a new instance of {@link Function}. + * @param functionName The name of the function. + * @param parameterTypes The type of the parameter(s). + * @param encapsulationFunction An encapsulation of the function's runtime computation. + */ + public Function(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final EncapsulationFunction encapsulationFunction) { + this(functionName, parameterTypes, null, encapsulationFunction); + } + + /** + * Creates a new instance of {@link Function}. + * @param functionName The name of the function. + * @param parameterTypes The type of the parameter(s). + * @param variadicSuffixType The type of the function's vararg. + * @param encapsulationFunction An encapsulation of the function's runtime computation. + */ + protected Function(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nullable final Type variadicSuffixType, @Nonnull final EncapsulationFunction encapsulationFunction) { + this.functionName = functionName; + this.parameterTypes = ImmutableList.copyOf(parameterTypes); + this.variadicSuffixType = variadicSuffixType; + this.encapsulationFunction = encapsulationFunction; + } + + @Nonnull + public String getFunctionName() { + return functionName; + } + + @Nonnull + public List getParameterTypes() { + return parameterTypes; + } + + @Nonnull + public Type resolveParameterType(int index) { + Verify.verify(index >= 0, "unexpected negative parameter index"); + if (index < parameterTypes.size()) { + return parameterTypes.get(index); + } else { + if (hasVariadicSuffix()) { + return variadicSuffixType; + } + throw new IllegalArgumentException("cannot resolve declared parameter at index " + index); + } + } + + @Nonnull + public List resolveParameterTypes(int numberOfArguments) { + Verify.verify(numberOfArguments > 0, "unexpected number of arguments"); + final ImmutableList.Builder resultBuilder = ImmutableList.builder(); + for (int i = 0; i < numberOfArguments; i ++) { + resultBuilder.add(resolveParameterType(i)); + } + return resultBuilder.build(); + } + + @Nullable + public Type getVariadicSuffixType() { + return variadicSuffixType; + } + + public boolean hasVariadicSuffix() { + return variadicSuffixType != null; + } + + @Nonnull + public EncapsulationFunction getEncapsulationFunction() { + return encapsulationFunction; + } + + @Nonnull + // in Function abstract encapsulate() + public abstract T encapsulate(@Nonnull final List arguments); + + @Nonnull + @Override + public String toString() { + String variadicSuffixString = ""; + if (variadicSuffixType != null) { + variadicSuffixString = variadicSuffixType + "..."; + if (!parameterTypes.isEmpty()) { + variadicSuffixString = ", " + variadicSuffixString; + } + } + + return functionName + "(" + parameterTypes.stream().map(Object::toString).collect(Collectors.joining(",")) + variadicSuffixString + ")"; + } +} From d058c09d181fe3141c657eeb92202d003f61c3b7 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 9 Jan 2025 16:47:07 -0800 Subject: [PATCH 15/41] MacroFunction --- .../record/RecordMetaDataBuilder.java | 5 +- .../foundationdb/record/metadata/Udf.java | 21 +-- .../query/plan/cascades/BuiltInFunction.java | 16 +- .../record/query/plan/cascades/Function.java | 31 +--- .../query/plan/cascades/MacroFunction.java | 92 ++++++++++ .../cascades/values/JavaCallFunction.java | 2 +- .../cascades/values/MacroFunctionValue.java | 158 ------------------ .../plan/cascades/values/UdfFunction.java | 2 +- .../src/main/proto/record_metadata.proto | 5 +- .../src/main/proto/record_query_plan.proto | 5 +- gradle/strict.gradle | 1 + 11 files changed, 123 insertions(+), 215 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java delete mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index ee53fc2f9e..2aae254bda 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -43,7 +43,7 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; import com.apple.foundationdb.record.provider.foundationdb.MetaDataProtoEditor; -import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.MacroFunction; import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; import com.google.common.base.Verify; import com.google.common.collect.ImmutableMap; @@ -232,7 +232,8 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, PlanHashable.CURRENT_FOR_CONTINUATION); for (RecordMetaDataProto.Udf udf: metaDataProto.getUdfsList()) { - udfMap.put(udf.getFunctionName(), new Udf(udf.getFunctionName(), Value.fromValueProto(serializationContext, udf.getFunctionValue()))); + MacroFunction function = MacroFunction.fromProto(serializationContext, udf.getFunctionValue()); + udfMap.put(function.getFunctionName(), new Udf(function)); } if (metaDataProto.hasSplitLongRecords()) { splitLongRecords = metaDataProto.getSplitLongRecords(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java index b7be2ddf44..d65472a5e4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.PlanSerializationContext; import com.apple.foundationdb.record.RecordMetaDataProto; -import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.MacroFunction; import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; import javax.annotation.Nonnull; @@ -32,32 +32,27 @@ * Defines a User-defined-function. */ public class Udf { - @Nonnull private final String functionName; @Nonnull - private final Value functionValue; + private final MacroFunction macroFunction; - public Udf(@Nonnull String functionName, @Nonnull Value functionValue) { - this.functionName = functionName; - this.functionValue = functionValue; + public Udf(@Nonnull MacroFunction functionValue) { + this.macroFunction = functionValue; } @Nonnull - public String getFunctionName() { - return functionName; + public MacroFunction getMacroFunction() { + return macroFunction; } @Nonnull - public Value getValue() { - return functionValue; - } + public String getFunctionName() {return macroFunction.getFunctionName();} @Nonnull public RecordMetaDataProto.Udf toProto() { PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, PlanHashable.CURRENT_FOR_CONTINUATION); return RecordMetaDataProto.Udf.newBuilder() - .setFunctionName(functionName) - .setFunctionValue(functionValue.toValueProto(serializationContext)) + .setFunctionValue(macroFunction.toProto(serializationContext)) .build(); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java index dc15304c38..f604752716 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java @@ -22,15 +22,12 @@ import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; -import com.google.common.base.Verify; -import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; /** * Main interface for defining a built-in function that can be evaluated against a number of arguments. @@ -42,6 +39,8 @@ */ @SuppressWarnings("PMD.AbstractClassWithoutAbstractMethod") public abstract class BuiltInFunction extends Function{ + @Nonnull + private final EncapsulationFunction encapsulationFunction; /** * Creates a new instance of {@link BuiltInFunction}. * @param functionName The name of the function. @@ -49,7 +48,8 @@ public abstract class BuiltInFunction extends Function{ * @param encapsulationFunction An encapsulation of the function's runtime computation. */ public BuiltInFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final EncapsulationFunction encapsulationFunction) { - super(functionName, parameterTypes, null, encapsulationFunction); + super(functionName, parameterTypes, null); + this.encapsulationFunction = encapsulationFunction; } /** @@ -60,7 +60,13 @@ public BuiltInFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nullable final Type variadicSuffixType, @Nonnull final EncapsulationFunction encapsulationFunction) { - super(functionName, parameterTypes, variadicSuffixType, encapsulationFunction); + super(functionName, parameterTypes, variadicSuffixType); + this.encapsulationFunction = encapsulationFunction; + } + + @Nonnull + public EncapsulationFunction getEncapsulationFunction() { + return encapsulationFunction; } /** diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java index 26efd4a92c..cde10e2d17 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java @@ -28,16 +28,10 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.List; -import java.util.Objects; -import java.util.Optional; import java.util.stream.Collectors; /** - * Main interface for defining a built-in function that can be evaluated against a number of arguments. - * - * A function could have a fixed number of arguments such as Value Add(Value, Value), or a - * variable number of arguments, such as Value TEXT_CONTAINS_ALL_PREFIXES(Value, Value, Value, Value, ...). - * + * Main interface for defining a function that can be evaluated against a number of arguments. * @param The resulting type of the function. */ @SuppressWarnings("PMD.AbstractClassWithoutAbstractMethod") @@ -54,31 +48,16 @@ public abstract class Function{ @Nullable final Type variadicSuffixType; - @Nonnull - final EncapsulationFunction encapsulationFunction; - - /** - * Creates a new instance of {@link Function}. - * @param functionName The name of the function. - * @param parameterTypes The type of the parameter(s). - * @param encapsulationFunction An encapsulation of the function's runtime computation. - */ - public Function(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final EncapsulationFunction encapsulationFunction) { - this(functionName, parameterTypes, null, encapsulationFunction); - } - /** * Creates a new instance of {@link Function}. * @param functionName The name of the function. * @param parameterTypes The type of the parameter(s). * @param variadicSuffixType The type of the function's vararg. - * @param encapsulationFunction An encapsulation of the function's runtime computation. */ - protected Function(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nullable final Type variadicSuffixType, @Nonnull final EncapsulationFunction encapsulationFunction) { + public Function(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nullable final Type variadicSuffixType) { this.functionName = functionName; this.parameterTypes = ImmutableList.copyOf(parameterTypes); this.variadicSuffixType = variadicSuffixType; - this.encapsulationFunction = encapsulationFunction; } @Nonnull @@ -124,12 +103,6 @@ public boolean hasVariadicSuffix() { } @Nonnull - public EncapsulationFunction getEncapsulationFunction() { - return encapsulationFunction; - } - - @Nonnull - // in Function abstract encapsulate() public abstract T encapsulate(@Nonnull final List arguments); @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java new file mode 100644 index 0000000000..9b4dd6af6b --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java @@ -0,0 +1,92 @@ +/* + * MacroFunction.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades; + +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.planprotos.PMacroFunctionValue; +import com.apple.foundationdb.record.planprotos.PValue; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; +import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.google.common.collect.ImmutableList; + +import javax.annotation.Nonnull; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Main interface for defining a user-defined function that can be evaluated against a number of arguments. + */ +public class MacroFunction extends Function{ + @Nonnull + private final com.apple.foundationdb.record.query.plan.cascades.values.Value body; + private final List identifiers; + + public MacroFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final List identifiers, @Nonnull final com.apple.foundationdb.record.query.plan.cascades.values.Value body) { + super(functionName, parameterTypes, null); + this.identifiers = ImmutableList.copyOf(identifiers); + this.body = body; + } + + @Nonnull + @Override + public com.apple.foundationdb.record.query.plan.cascades.values.Value encapsulate(@Nonnull List arguments) { + // replace the QuantifiedObjectValue in body with arguments + SemanticException.check(arguments.size() == parameterTypes.size(), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument length doesn't match with function definition"); + TranslationMap.Builder translationMapBuilder = new TranslationMap.Builder(); + for (int i = 0; i < arguments.size(); i++) { + // check that arguments[i] type matches with parameterTypes[i] + final int finalI = i; + SemanticException.check(arguments.get(finalI).getResultType().equals(parameterTypes.get(i)), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument type doesn't match with function definition"); + translationMapBuilder.when(identifiers.get(finalI)).then((sourceAlias, leafValue) -> (Value)arguments.get(finalI)); + } + return body.translateCorrelations(translationMapBuilder.build()); + } + + @Nonnull + public PMacroFunctionValue toProto(@Nonnull final PlanSerializationContext serializationContext) { + PMacroFunctionValue.Builder builder = PMacroFunctionValue.newBuilder(); + for (int i = 0; i < parameterTypes.size(); i++) { + builder.addArguments(QuantifiedObjectValue.of(identifiers.get(i), parameterTypes.get(i)).toValueProto(serializationContext)); + } + return builder + .setFunctionName(functionName) + .setBody(body.toValueProto(serializationContext)) + .build(); + } + + + @Nonnull + public PValue toValueProto(@Nonnull final PlanSerializationContext serializationContext) { + final var specificValueProto = toProto(serializationContext); + return PValue.newBuilder().setMacroFunctionValue(specificValueProto).build(); + } + + @Nonnull + public static MacroFunction fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PMacroFunctionValue functionValue) { + return new MacroFunction(functionValue.getFunctionName(), + functionValue.getArgumentsList().stream().map(pvalue -> Value.fromValueProto(serializationContext, pvalue).getResultType()).collect(Collectors.toList()), + functionValue.getArgumentsList().stream().map(pvalue -> ((QuantifiedObjectValue)Value.fromValueProto(serializationContext, pvalue)).getAlias()).collect(Collectors.toList()), + Value.fromValueProto(serializationContext, functionValue.getBody())); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/JavaCallFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/JavaCallFunction.java index 407985f624..4b2094c729 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/JavaCallFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/JavaCallFunction.java @@ -72,7 +72,7 @@ private static Value findFunction(@Nonnull final BuiltInFunction ignored, // the class must have parameterless constructor try { final Constructor constructor = clazz.getDeclaredConstructor(); - return (Value)((UdfFunction)constructor.newInstance()).encapsulate(arguments.stream().skip(1).collect(Collectors.toUnmodifiableList())); + return ((UdfFunction)constructor.newInstance()).encapsulate(arguments.stream().skip(1).collect(Collectors.toUnmodifiableList())); } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) { throw new RecordCoreException("could not instantiate call-site from '" + clazz.getName() + "'", e); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java deleted file mode 100644 index b7c485c0b3..0000000000 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/MacroFunctionValue.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * MacroFunctionValue.java - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apple.foundationdb.record.query.plan.cascades.values; - -import com.apple.foundationdb.record.EvaluationContext; -import com.apple.foundationdb.record.PlanDeserializer; -import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.PlanSerializationContext; -import com.apple.foundationdb.record.planprotos.PMacroFunctionValue; -import com.apple.foundationdb.record.planprotos.PValue; -import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; -import com.apple.foundationdb.record.query.plan.cascades.SemanticException; -import com.apple.foundationdb.record.query.plan.cascades.typing.Type; -import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; -import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; -import com.google.auto.service.AutoService; -import com.google.common.collect.ImmutableList; -import com.google.protobuf.Message; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.util.List; -import java.util.stream.Collectors; - -/** - * The class defines a function. - * The function takes in a list of arguments as QuantifiedObjectValue, of which alias = uniqueId, resultType = argument type. - * and a body value, executing the function is by replacing the QuantifiedObjectValue in the body with the actual argument. - */ -public class MacroFunctionValue extends AbstractValue { - - @Nonnull - private final List argList; - - @Nonnull - private final Value body; - - private MacroFunctionValue(@Nonnull final List argList, @Nonnull final Value underlying) { - this.argList = argList; - this.body = underlying; - } - - @Override - public int planHash(@Nonnull final PlanHashMode hashKind) { - return PlanHashable.objectsPlanHash(hashKind, argList, body); - } - - @Override - @Nonnull - public Type getResultType() { - return body.getResultType(); - } - - @Nonnull - public Value getBody() { - return body; - } - - @Nullable - @Override - public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { - return body.eval(store, context); - } - - @Nonnull - public static MacroFunctionValue of(@Nonnull final List argList, @Nonnull final Value body) { - return new MacroFunctionValue(argList, body); - } - - @Nullable - public Value call(@Nonnull List arguments) { - // replace the QuantifiedObjectValue in body with arguments - SemanticException.check(arguments.size() == argList.size(), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument length doesn't match with function definition"); - TranslationMap.Builder translationMapBuilder = new TranslationMap.Builder(); - for (int i = 0; i < arguments.size(); i++) { - // check that arguments[i] type matches with argList[i] type - final int finalI = i; - SemanticException.check(arguments.get(finalI).getResultType().equals(argList.get(i).getResultType()), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument type doesn't match with function definition"); - translationMapBuilder.when(((QuantifiedObjectValue) argList.get(i)).getAlias()).then((sourceAlias, leafValue) -> (Value)arguments.get(finalI)); - } - return body.translateCorrelations(translationMapBuilder.build()); - } - - @Override - public int hashCodeWithoutChildren() { - return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, this.getClass().getCanonicalName()); - } - - @Nonnull - @Override - public PMacroFunctionValue toProto(@Nonnull final PlanSerializationContext serializationContext) { - PMacroFunctionValue.Builder builder = PMacroFunctionValue.newBuilder(); - argList.forEach(arg -> builder.addArguments(arg.toValueProto(serializationContext))); - return builder.setBody(body.toValueProto(serializationContext)).build(); - } - - - @Nonnull - @Override - public PValue toValueProto(@Nonnull final PlanSerializationContext serializationContext) { - final var specificValueProto = toProto(serializationContext); - return PValue.newBuilder().setMacroFunctionValue(specificValueProto).build(); - } - - @Nonnull - public static MacroFunctionValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PMacroFunctionValue functionValue) { - return new MacroFunctionValue(functionValue.getArgumentsList().stream().map(pvalue -> Value.fromValueProto(serializationContext, pvalue)).collect(Collectors.toList()), Value.fromValueProto(serializationContext, functionValue.getBody())); - } - - @Nonnull - @Override - protected Iterable computeChildren() { - return ImmutableList.of(); - } - - @Nonnull - @Override - public Value withChildren(final Iterable newChildren) { - return this; - } - - /** - * Deserializer. - */ - @AutoService(PlanDeserializer.class) - public static class Deserializer implements PlanDeserializer { - @Nonnull - @Override - public Class getProtoMessageClass() { - return PMacroFunctionValue.class; - } - - @Nonnull - @Override - public MacroFunctionValue fromProto(@Nonnull final PlanSerializationContext serializationContext, - @Nonnull final PMacroFunctionValue functionValue) { - return MacroFunctionValue.fromProto(serializationContext, functionValue); - } - } -} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/UdfFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/UdfFunction.java index 0baebe73ab..f8cc15ef7e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/UdfFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/UdfFunction.java @@ -57,7 +57,7 @@ public final String getFunctionName() { @SuppressWarnings("unchecked") @Nonnull @Override - public final Typed encapsulate(@Nonnull final List arguments) { + public final Value encapsulate(@Nonnull final List arguments) { arguments.forEach(argument -> Verify.verify(argument instanceof Value)); final List parameterTypes = getParameterTypes(); if (arguments.size() != parameterTypes.size()) { diff --git a/fdb-record-layer-core/src/main/proto/record_metadata.proto b/fdb-record-layer-core/src/main/proto/record_metadata.proto index 832acb465e..d2eb23b1ff 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata.proto @@ -104,12 +104,9 @@ message MetaData { } message Udf { - optional string function_name = 1; - optional com.apple.foundationdb.record.planprotos.PValue function_value = 2; + optional com.apple.foundationdb.record.planprotos.PMacroFunctionValue function_value = 1; } - - message JoinedRecordType { optional string name = 1; optional Value record_type_key = 4; diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index ee3e8e4d4b..9370202a12 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -247,8 +247,9 @@ message PValue { } message PMacroFunctionValue { - repeated PValue arguments = 1; - optional PValue body = 2; + optional string function_name = 1; + repeated PValue arguments = 2; + optional PValue body = 3; } message PAbstractArrayConstructorValue { diff --git a/gradle/strict.gradle b/gradle/strict.gradle index 5b2366b274..cfefcb334b 100644 --- a/gradle/strict.gradle +++ b/gradle/strict.gradle @@ -23,6 +23,7 @@ tasks.withType(JavaCompile) { // suppress static due to protoc generating super.addAll calls. options.compilerArgs += ["-Xlint:all", "-Xlint:-processing", "-Xlint:-options", "-Xlint:-static", "-Werror"] + options.compilerArgs.remove('-Werror') } spotbugs { From 67fad068f5ca0e9ae5509028d34a35a96c5293a9 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Fri, 10 Jan 2025 10:20:44 -0800 Subject: [PATCH 16/41] save --- .../foundationdb/record/RecordMetaData.java | 18 +++++++---------- .../record/RecordMetaDataBuilder.java | 20 +++++++------------ .../foundationdb/record/metadata/Udf.java | 9 ++++++++- .../record/query/expressions/Comparisons.java | 11 +++++----- .../plan/IndexKeyValueToPartialRecord.java | 11 +++++----- .../query/plan/cascades/MacroFunction.java | 18 ++++++++--------- 6 files changed, 43 insertions(+), 44 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java index f01fe23205..81ea4e218e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java @@ -49,6 +49,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -84,7 +85,7 @@ public class RecordMetaData implements RecordMetaDataProvider { @Nonnull private final Map> recordTypeKeyToSyntheticTypeMap; @Nonnull - private final Map udfMap; + private final Set udfs; @Nonnull private final Map indexes; @Nonnull @@ -115,7 +116,7 @@ protected RecordMetaData(@Nonnull RecordMetaData orig) { Collections.unmodifiableMap(orig.indexes), Collections.unmodifiableMap(orig.universalIndexes), Collections.unmodifiableList(orig.formerIndexes), - Collections.unmodifiableMap(orig.udfMap), + Collections.unmodifiableSet(orig.udfs), orig.splitLongRecords, orig.storeRecordVersions, orig.version, @@ -135,7 +136,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor, @Nonnull Map indexes, @Nonnull Map universalIndexes, @Nonnull List formerIndexes, - @Nonnull Map udfMap, + @Nonnull Set udfs, boolean splitLongRecords, boolean storeRecordVersions, int version, @@ -152,7 +153,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor, this.indexes = indexes; this.universalIndexes = universalIndexes; this.formerIndexes = formerIndexes; - this.udfMap = udfMap; + this.udfs = udfs; this.splitLongRecords = splitLongRecords; this.storeRecordVersions = storeRecordVersions; this.version = version; @@ -348,14 +349,9 @@ public List getFormerIndexes() { return formerIndexes; } - @Nullable - public Udf getUdf(@Nonnull String name) { - return udfMap.get(name); - } - @Nonnull public Collection getAllUdfs() { - return udfMap.values(); + return udfs; } public boolean isSplitLongRecords() { @@ -709,7 +705,7 @@ public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor } // Add in the final options. - builder.addAllUdfs(udfMap.values().stream().map(Udf::toProto).collect(Collectors.toList())); + builder.addAllUdfs(udfs.stream().map(Udf::toProto).collect(Collectors.toList())); builder.setSplitLongRecords(splitLongRecords); builder.setStoreRecordVersions(storeRecordVersions); builder.setVersion(version); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index 2aae254bda..1c47cdcb97 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -43,8 +43,6 @@ import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistry; import com.apple.foundationdb.record.provider.foundationdb.IndexMaintainerRegistryImpl; import com.apple.foundationdb.record.provider.foundationdb.MetaDataProtoEditor; -import com.apple.foundationdb.record.query.plan.cascades.MacroFunction; -import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; import com.google.common.base.Verify; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Maps; @@ -56,7 +54,6 @@ import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; @@ -114,7 +111,7 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider { @Nonnull private final Map> syntheticRecordTypes; @Nonnull - private final Map udfMap; + private final Set udfs; @Nonnull private final Map indexes; @Nonnull @@ -150,7 +147,7 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider { indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance(); evolutionValidator = MetaDataEvolutionValidator.getDefaultInstance(); syntheticRecordTypes = new HashMap<>(); - udfMap = new HashMap<>(); + udfs = new HashSet<>(); } private void processSchemaOptions(boolean processExtensionOptions) { @@ -229,11 +226,8 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey())); } } - PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, - PlanHashable.CURRENT_FOR_CONTINUATION); for (RecordMetaDataProto.Udf udf: metaDataProto.getUdfsList()) { - MacroFunction function = MacroFunction.fromProto(serializationContext, udf.getFunctionValue()); - udfMap.put(function.getFunctionName(), new Udf(function)); + udfs.add(Udf.fromProto(udf)); } if (metaDataProto.hasSplitLongRecords()) { splitLongRecords = metaDataProto.getSplitLongRecords(); @@ -1193,11 +1187,11 @@ public void addFormerIndex(@Nonnull FormerIndex formerIndex) { } public void addUdf(@Nonnull Udf udf) { - udfMap.put(udf.getFunctionName(), udf); + udfs.add(udf); } - public void addUdfs(@Nonnull Collection udfs) { - udfs.forEach(udf -> udfMap.put(udf.getFunctionName(), udf)); + public void addUdfs(@Nonnull Iterable udfs) { + udfs.forEach(udf -> this.udfs.add(udf)); } public boolean isSplitLongRecords() { @@ -1441,7 +1435,7 @@ public RecordMetaData build(boolean validate) { Map> recordTypeKeyToSyntheticRecordTypeMap = Maps.newHashMapWithExpectedSize(syntheticRecordTypes.size()); RecordMetaData metaData = new RecordMetaData(recordsDescriptor, getUnionDescriptor(), unionFields, builtRecordTypes, builtSyntheticRecordTypes, recordTypeKeyToSyntheticRecordTypeMap, - indexes, universalIndexes, formerIndexes, udfMap, + indexes, universalIndexes, formerIndexes, udfs, splitLongRecords, storeRecordVersions, version, subspaceKeyCounter, usesSubspaceKeyCounter, recordCountKey, localFileDescriptor != null); for (RecordTypeBuilder recordTypeBuilder : recordTypes.values()) { KeyExpression primaryKey = recordTypeBuilder.getPrimaryKey(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java index d65472a5e4..b089be2dba 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java @@ -3,7 +3,7 @@ * * This source file is part of the FoundationDB open source project * - * Copyright 2015-2024 Apple Inc. and the FoundationDB project authors + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -55,4 +55,11 @@ public RecordMetaDataProto.Udf toProto() { .setFunctionValue(macroFunction.toProto(serializationContext)) .build(); } + + @Nonnull + public static Udf fromProto(RecordMetaDataProto.Udf proto) { + PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, + PlanHashable.CURRENT_FOR_CONTINUATION); + return new Udf(MacroFunction.fromProto(serializationContext, proto.getFunctionValue())); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java index 2076f62759..afaf625b38 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/expressions/Comparisons.java @@ -38,6 +38,7 @@ import com.apple.foundationdb.record.metadata.expressions.InvertibleFunctionKeyExpression; import com.apple.foundationdb.record.metadata.expressions.TupleFieldsHelper; import com.apple.foundationdb.record.planprotos.PComparison; +import com.apple.foundationdb.record.planprotos.PComparison.PComparisonType; import com.apple.foundationdb.record.planprotos.PInvertedFunctionComparison; import com.apple.foundationdb.record.planprotos.PListComparison; import com.apple.foundationdb.record.planprotos.PMultiColumnComparison; @@ -634,8 +635,8 @@ public enum Type { LIKE; @Nonnull - private static final Supplier> protoEnumBiMapSupplier = - Suppliers.memoize(() -> PlanSerialization.protoEnumBiMap(Type.class, PComparison.PComparisonType.class)); + private static final Supplier> protoEnumBiMapSupplier = + Suppliers.memoize(() -> PlanSerialization.protoEnumBiMap(Type.class, PComparisonType.class)); private final boolean isEquality; private final boolean isUnary; @@ -663,19 +664,19 @@ public boolean isUnary() { @Nonnull @SuppressWarnings("unused") - public PComparison.PComparisonType toProto(@Nonnull final PlanSerializationContext serializationContext) { + public PComparisonType toProto(@Nonnull final PlanSerializationContext serializationContext) { return Objects.requireNonNull(getProtoEnumBiMap().get(this)); } @Nonnull @SuppressWarnings("unused") public static Type fromProto(@Nonnull final PlanSerializationContext serializationContext, - @Nonnull final PComparison.PComparisonType physicalOperatorProto) { + @Nonnull final PComparisonType physicalOperatorProto) { return Objects.requireNonNull(getProtoEnumBiMap().inverse().get(physicalOperatorProto)); } @Nonnull - private static BiMap getProtoEnumBiMap() { + private static BiMap getProtoEnumBiMap() { return protoEnumBiMapSupplier.get(); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java index a68a73abbe..c5a8ddee35 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/IndexKeyValueToPartialRecord.java @@ -40,6 +40,7 @@ import com.apple.foundationdb.record.planprotos.PIndexKeyValueToPartialRecord.PFieldCopier; import com.apple.foundationdb.record.planprotos.PIndexKeyValueToPartialRecord.PFieldWithValueCopier; import com.apple.foundationdb.record.planprotos.PIndexKeyValueToPartialRecord.PMessageCopier; +import com.apple.foundationdb.record.planprotos.PIndexKeyValueToPartialRecord.PTupleSource; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.predicates.ConstantPredicate; @@ -194,14 +195,14 @@ public enum TupleSource { @Nonnull @SuppressWarnings("unused") - public PIndexKeyValueToPartialRecord.PTupleSource toProto(@Nonnull final PlanSerializationContext serializationContext) { + public PTupleSource toProto(@Nonnull final PlanSerializationContext serializationContext) { switch (this) { case KEY: - return PIndexKeyValueToPartialRecord.PTupleSource.KEY; + return PTupleSource.KEY; case VALUE: - return PIndexKeyValueToPartialRecord.PTupleSource.VALUE; + return PTupleSource.VALUE; case OTHER: - return PIndexKeyValueToPartialRecord.PTupleSource.OTHER; + return PTupleSource.OTHER; default: throw new RecordCoreException("unknown tuple source mapping. did you forget to add it?"); } @@ -210,7 +211,7 @@ public PIndexKeyValueToPartialRecord.PTupleSource toProto(@Nonnull final PlanSer @Nonnull @SuppressWarnings("unused") public static TupleSource fromProto(@Nonnull final PlanSerializationContext serializationContext, - @Nonnull final PIndexKeyValueToPartialRecord.PTupleSource tupleSourceProto) { + @Nonnull final PTupleSource tupleSourceProto) { switch (tupleSourceProto) { case KEY: return KEY; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java index 9b4dd6af6b..bb7c69ee38 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java @@ -39,13 +39,13 @@ */ public class MacroFunction extends Function{ @Nonnull - private final com.apple.foundationdb.record.query.plan.cascades.values.Value body; - private final List identifiers; + private final com.apple.foundationdb.record.query.plan.cascades.values.Value bodyValue; + private final List parameterIdentifiers; - public MacroFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final List identifiers, @Nonnull final com.apple.foundationdb.record.query.plan.cascades.values.Value body) { + public MacroFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final List parameterIdentifiers, @Nonnull final com.apple.foundationdb.record.query.plan.cascades.values.Value bodyValue) { super(functionName, parameterTypes, null); - this.identifiers = ImmutableList.copyOf(identifiers); - this.body = body; + this.parameterIdentifiers = ImmutableList.copyOf(parameterIdentifiers); + this.bodyValue = bodyValue; } @Nonnull @@ -58,20 +58,20 @@ public com.apple.foundationdb.record.query.plan.cascades.values.Value encapsulat // check that arguments[i] type matches with parameterTypes[i] final int finalI = i; SemanticException.check(arguments.get(finalI).getResultType().equals(parameterTypes.get(i)), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument type doesn't match with function definition"); - translationMapBuilder.when(identifiers.get(finalI)).then((sourceAlias, leafValue) -> (Value)arguments.get(finalI)); + translationMapBuilder.when(parameterIdentifiers.get(finalI)).then((sourceAlias, leafValue) -> (Value)arguments.get(finalI)); } - return body.translateCorrelations(translationMapBuilder.build()); + return bodyValue.translateCorrelations(translationMapBuilder.build()); } @Nonnull public PMacroFunctionValue toProto(@Nonnull final PlanSerializationContext serializationContext) { PMacroFunctionValue.Builder builder = PMacroFunctionValue.newBuilder(); for (int i = 0; i < parameterTypes.size(); i++) { - builder.addArguments(QuantifiedObjectValue.of(identifiers.get(i), parameterTypes.get(i)).toValueProto(serializationContext)); + builder.addArguments(QuantifiedObjectValue.of(parameterIdentifiers.get(i), parameterTypes.get(i)).toValueProto(serializationContext)); } return builder .setFunctionName(functionName) - .setBody(body.toValueProto(serializationContext)) + .setBody(bodyValue.toValueProto(serializationContext)) .build(); } From 3adcce36fde3d5de3726fad27704fa7a3ea7dcce Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Mon, 18 Nov 2024 15:45:16 -0800 Subject: [PATCH 17/41] save --- .../relational/api/metadata/DataType.java | 23 ++++++- .../api/metadata/FunctionDefinition.java | 30 +++++++++ .../src/main/antlr/RelationalLexer.g4 | 2 + .../src/main/antlr/RelationalParser.g4 | 7 +- ...RecordLayerStoreSchemaTemplateCatalog.java | 2 + .../metadata/RecordLayerSchemaTemplate.java | 27 +++++++- .../metadata/RecordLayerTable.java | 23 ++++++- .../RecordLayerUserDefinedFunction.java | 64 +++++++++++++++++++ .../recordlayer/query/PlanGenerator.java | 1 + .../query/visitors/BaseVisitor.java | 12 ++++ .../query/visitors/DdlVisitor.java | 18 ++++++ .../query/visitors/DelegatingVisitor.java | 12 ++++ .../query/visitors/ExpressionVisitor.java | 52 +++++++++++++++ .../query/visitors/IdentifierVisitor.java | 1 + .../query/visitors/TypedVisitor.java | 4 ++ .../relational/api/DriverManagerTest.java | 5 +- .../recordlayer/query/StandardQueryTests.java | 33 ++++++++++ 17 files changed, 308 insertions(+), 8 deletions(-) create mode 100644 fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/FunctionDefinition.java create mode 100644 fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerUserDefinedFunction.java diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java index 96ed59a281..cb95725d6e 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java @@ -29,6 +29,7 @@ import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import javax.annotation.Nonnull; import java.sql.Types; @@ -956,13 +957,17 @@ public static final class StructType extends DataType implements Named { @Nonnull private final String name; + @Nonnull + private final Map userDefinedFunctions; + @Nonnull private final Supplier resolvedSupplier = Suppliers.memoize(this::calculateResolved); - private StructType(@Nonnull final String name, boolean isNullable, @Nonnull final List fields) { + private StructType(@Nonnull final String name, boolean isNullable, @Nonnull final List fields, @Nonnull final Map userDefinedOperators) { super(isNullable, false, Code.STRUCT); this.name = name; this.fields = ImmutableList.copyOf(fields); + this.userDefinedFunctions = ImmutableMap.copyOf(userDefinedOperators); } @Nonnull @@ -976,6 +981,11 @@ public String getName() { return name; } + @Nonnull + public Map getUserDefinedFunctions() { + return ImmutableMap.copyOf(userDefinedFunctions); + } + public static class Field { @Nonnull private final Supplier hashCodeSupplier = Suppliers.memoize(this::computeHashCode); @@ -1046,7 +1056,14 @@ public String toString() { @Nonnull public static StructType from(@Nonnull final String name, @Nonnull final List fields, boolean isNullable) { - return new StructType(name, isNullable, fields); + return new StructType(name, isNullable, fields, ImmutableMap.of()); + } + + @Nonnull + public StructType withFunctionDefinition(FunctionDefinition functionDefinition) { + Map newUserDefinedFunctions = new java.util.HashMap<>(userDefinedFunctions); + newUserDefinedFunctions.put(functionDefinition.getName(), functionDefinition); + return new StructType(name, isNullable(), fields, newUserDefinedFunctions); } @Override @@ -1055,7 +1072,7 @@ public StructType withNullable(boolean isNullable) { if (isNullable == isNullable()) { return this; } - return new StructType(name, isNullable, fields); + return new StructType(name, isNullable, fields, userDefinedFunctions); } private boolean calculateResolved() { diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/FunctionDefinition.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/FunctionDefinition.java new file mode 100644 index 0000000000..d42568f484 --- /dev/null +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/FunctionDefinition.java @@ -0,0 +1,30 @@ +/* + * FunctionDefinition.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.api.metadata; + +import javax.annotation.Nonnull; + +public interface FunctionDefinition extends Metadata{ + @Override + default void accept(@Nonnull final Visitor visitor) { + visitor.visit(this); + } +} diff --git a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 index 72879bb762..29802d8b3c 100644 --- a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 @@ -527,6 +527,7 @@ KEY_BLOCK_SIZE: 'KEY_BLOCK_SIZE'; LANGUAGE: 'LANGUAGE'; LAST: 'LAST'; LEAVES: 'LEAVES'; +LEFTARG: 'LEFTARG'; LESS: 'LESS'; LEVEL: 'LEVEL'; LIST: 'LIST'; @@ -646,6 +647,7 @@ RESUME: 'RESUME'; RETURNED_SQLSTATE: 'RETURNED_SQLSTATE'; RETURNING: 'RETURNING'; RETURNS: 'RETURNS'; +RIGHTARG: 'RIGHTARG'; ROLE: 'ROLE'; ROLLBACK: 'ROLLBACK'; ROLLUP: 'ROLLUP'; diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index 973bd18f9a..c625e3a1d2 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -88,7 +88,7 @@ utilityStatement templateClause : - CREATE ( structDefinition | tableDefinition | enumDefinition | indexDefinition ) + CREATE ( structDefinition | tableDefinition | enumDefinition | indexDefinition | functionDefinition) ; createStatement @@ -154,6 +154,10 @@ indexDefinition : (UNIQUE)? INDEX indexName=uid AS queryTerm indexAttributes? ; +functionDefinition + : FUNCTION functionName=uid LEFT_ROUND_BRACKET paramName=uid inputTypeName=uid RIGHT_ROUND_BRACKET RETURNS LEFT_ROUND_BRACKET columnType ( ',' columnType)* RIGHT_ROUND_BRACKET AS LEFT_ROUND_BRACKET expression ( ',' expression)* RIGHT_ROUND_BRACKET + ; + indexAttributes : WITH ATTRIBUTES indexAttribute (COMMA indexAttribute)* ; @@ -789,6 +793,7 @@ functionCall : aggregateWindowedFunction #aggregateFunctionCall // done (supported) | specificFunction #specificFunctionCall // | scalarFunctionName '(' functionArgs? ')' #scalarFunctionCall // done (unsupported) + | userDefinedFunctionName=uid '(' functionArgs? ')' #userDefinedFunctionCall ; specificFunction diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/RecordLayerStoreSchemaTemplateCatalog.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/RecordLayerStoreSchemaTemplateCatalog.java index 95fb7b0434..03c3e69a2e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/RecordLayerStoreSchemaTemplateCatalog.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/RecordLayerStoreSchemaTemplateCatalog.java @@ -160,6 +160,7 @@ private static Tuple getSchemaTemplatePrimaryKey(String schemaTemplateName) { @Override public SchemaTemplate loadSchemaTemplate(@Nonnull Transaction txn, @Nonnull String templateName) throws RelationalException { + System.out.println("RLStoreSchemaTemplateCatalog loadSchemaTemplate called with templateName:" + templateName); Tuple key = getSchemaTemplatePrimaryKey(templateName); var recordStore = RecordLayerStoreUtils.openRecordStore(txn, this.catalogSchemaPath, this.catalogRecordMetaDataProvider); @@ -202,6 +203,7 @@ public SchemaTemplate loadSchemaTemplate(@Nonnull Transaction txn, @Nonnull Stri */ @Nonnull private static SchemaTemplate toSchemaTemplate(@Nonnull Message m) throws InvalidProtocolBufferException { + System.out.println("RecordLayerStoreSchemaTemplateCatalog::toSchemaTemplate called"); Descriptors.Descriptor d = m.getDescriptorForType(); ByteString bs = (ByteString) m.getField(d.findFieldByName(SchemaTemplateSystemTable.METADATA)); RecordMetaData metaData = RecordMetaData.build(RecordMetaDataProto.MetaData.parseFrom(bs.toByteArray())); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java index 9648123d52..48fd3d92f5 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java @@ -30,6 +30,7 @@ import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.api.metadata.Index; +import com.apple.foundationdb.relational.api.metadata.FunctionDefinition; import com.apple.foundationdb.relational.api.metadata.SchemaTemplate; import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.api.metadata.Visitor; @@ -48,11 +49,13 @@ import com.google.protobuf.Descriptors; import javax.annotation.Nonnull; +import javax.annotation.Nullable; import java.util.BitSet; import java.util.Collection; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -147,6 +150,19 @@ public RecordLayerSchema generateSchema(@Nonnull String databaseId, @Nonnull Str return new RecordLayerSchema(schemaName, databaseId, this); } + @Nullable + public FunctionDefinition getOperationDefinition(@Nonnull final String operatorName, @Nonnull final DataType.StructType structType) { + for (RecordLayerTable table: tables) { + for (RecordLayerColumn column: table.getColumns()) { + final var dt = column.getDataType(); + if (dt instanceof DataType.StructType && dt.equals(structType) && ((DataType.StructType) dt).getUserDefinedFunctions().containsKey(operatorName)) { + return ((DataType.StructType) dt).getUserDefinedFunctions().get(operatorName); + } + } + } + return null; + } + @Nonnull public Descriptors.Descriptor getDescriptor(@Nonnull final String tableName) { return toRecordMetadata().getRecordType(tableName).getDescriptor(); @@ -178,6 +194,7 @@ public RecordMetaData toRecordMetadata() { public static RecordLayerSchemaTemplate fromRecordMetadata(@Nonnull RecordMetaData metaData, @Nonnull String templateName, int version) { + System.out.println("RecordLayerSchemaTemplate.fromRecordMetadata is called"); final var deserializer = new RecordMetadataDeserializer(metaData); final var builder = deserializer.getSchemaTemplate(templateName, version); return builder.setCachedMetadata(metaData).build(); @@ -385,6 +402,7 @@ public Builder addAuxiliaryType(@Nonnull DataType.Named auxiliaryType) { Assert.thatUnchecked(!tables.containsKey(auxiliaryType.getName()), ErrorCode.INVALID_SCHEMA_TEMPLATE, TABLE_ALREADY_EXISTS, auxiliaryType.getName()); Assert.thatUnchecked(!auxiliaryTypes.containsKey(auxiliaryType.getName()), ErrorCode.INVALID_SCHEMA_TEMPLATE, TYPE_WITH_NAME_ALREADY_EXISTS, auxiliaryType.getName()); auxiliaryTypes.put(auxiliaryType.getName(), auxiliaryType); + System.out.println("auxiliaryTypes name:" + auxiliaryType.getName()); return this; } @@ -420,6 +438,13 @@ public RecordLayerTable extractTable(@Nonnull final String name) { return tables.remove(name); } + @Nonnull + public DataType.Named extractAuxiliaryType(@Nonnull final String name) { + System.out.println("extractAuxiliaryType called with:" + name + " map:" + auxiliaryTypes); + Assert.thatUnchecked(auxiliaryTypes.containsKey(name.toUpperCase(Locale.ROOT)), ErrorCode.UNDEFINED_TABLE, "could not find '%s'", name); + return auxiliaryTypes.remove(name.toUpperCase(Locale.ROOT)); + } + @Nonnull public Optional findType(@Nonnull final String name) { // we should also check whether the name exists in _both_ databases. @@ -459,7 +484,7 @@ public RecordLayerSchemaTemplate build() { if (needsResolution) { resolveTypes(); } - + System.out.println("RecordLayerSchemaTemplate name:" + name + " is build with auxiliary type size:" + auxiliaryTypes.size()); if (cachedMetadata != null) { return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), version, enableLongRows, storeRowVersions, cachedMetadata); } else { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java index 5a9170eb3a..805257788a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java @@ -63,6 +63,9 @@ public final class RecordLayerTable implements Table { @Nonnull private final Set indexes; + @Nonnull + private final Set operatorDefinitions; + @Nonnull private final KeyExpression primaryKey; @@ -78,6 +81,7 @@ public final class RecordLayerTable implements Table { private RecordLayerTable(@Nonnull final String name, @Nonnull final List columns, @Nonnull final Set indexes, + @Nonnull final Set operatorDefinitions, @Nonnull final KeyExpression primaryKey, @Nonnull final Map generations, final DataType.StructType dataType, @@ -85,6 +89,7 @@ private RecordLayerTable(@Nonnull final String name, this.name = name; this.columns = ImmutableList.copyOf(columns); this.indexes = ImmutableSet.copyOf(indexes); + this.operatorDefinitions = ImmutableSet.copyOf(operatorDefinitions); this.primaryKey = primaryKey; this.generations = generations; this.dataType = dataType == null ? calculateDataType() : dataType; @@ -103,6 +108,11 @@ public Set getIndexes() { return indexes; } + @Nonnull + public Set getOperatorDefinitions() { + return operatorDefinitions; + } + @Nonnull public Map getGenerations() { return generations; @@ -205,6 +215,9 @@ public static final class Builder { @Nonnull private Set indexes; + @Nonnull + private Set operatorDefinitions; + @Nonnull private ImmutableList.Builder columns; @@ -258,6 +271,13 @@ public Builder addIndex(@Nonnull final RecordLayerIndex index) { return this; } + @Nonnull + public Builder addOperatorDefinition(@Nonnull final RecordLayerUserDefinedFunction userDefinedOperator) { + Assert.thatUnchecked(operatorDefinitions.stream().noneMatch(i -> userDefinedOperator.getName().equals(i.getName())), ErrorCode.INDEX_ALREADY_EXISTS, "attempt to add duplicate index '%s'", userDefinedOperator.getName()); + this.operatorDefinitions.add(userDefinedOperator); + return this; + } + @Nonnull public Builder addGenerations(@Nonnull final Map generations) { generations.forEach(this::addGeneration); @@ -337,8 +357,9 @@ public RecordLayerTable build() { Assert.thatUnchecked(!columnsList.isEmpty(), ErrorCode.INVALID_TABLE_DEFINITION, "Attempting to create table %s without columns", name); final var indexesSet = ImmutableSet.copyOf(indexes); + final Set operatorDefinitionsSet = operatorDefinitions == null ? ImmutableSet.of() : ImmutableSet.copyOf(operatorDefinitions); - return new RecordLayerTable(name, columnsList, indexesSet, getPrimaryKey(), generations, dataType, record); + return new RecordLayerTable(name, columnsList, indexesSet, operatorDefinitionsSet, getPrimaryKey(), generations, dataType, record); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerUserDefinedFunction.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerUserDefinedFunction.java new file mode 100644 index 0000000000..0dcce36cc1 --- /dev/null +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerUserDefinedFunction.java @@ -0,0 +1,64 @@ +/* + * RecordLayerUserDefinedFunction.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.metadata; + +import com.apple.foundationdb.relational.api.metadata.FunctionDefinition; +import com.apple.foundationdb.relational.api.metadata.Visitor; +import com.apple.foundationdb.relational.generated.RelationalParser; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RuleContext; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.TerminalNode; + +import javax.annotation.Nonnull; +import java.util.List; + +public class RecordLayerUserDefinedFunction implements FunctionDefinition { + @Nonnull private final String functionName; + @Nonnull private final String inputTypeName; + @Nonnull private final List returnColumnTypes; + @Nonnull private final String param; + @Nonnull private final List functionCalls; + + public RecordLayerUserDefinedFunction(@Nonnull String functionName, @Nonnull String inputTypeName, @Nonnull List returnColumnTypes, @Nonnull String param, @Nonnull List functionCalls) { + this.functionName = functionName; + this.inputTypeName = inputTypeName; + this.returnColumnTypes = returnColumnTypes; + this.param = param; + this.functionCalls = functionCalls; + } + + @Nonnull + @Override + public String getName() { + return functionName; + } + + @Override + public void accept(@Nonnull Visitor visitor) { + visitor.visit(this); + } + + @Nonnull + public String getInputTypeName() { + return inputTypeName; + } +} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java index 586508d718..9ca7efcf11 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java @@ -153,6 +153,7 @@ private Plan getPlanInternal(@Nonnull String query, @Nonnull KeyValueLogMessa QueryPlan.PhysicalQueryPlan.getValidPlanHashModes(options); final PlanHashable.PlanHashMode currentPlanHashMode = QueryPlan.PhysicalQueryPlan.getCurrentPlanHashMode(options); final var astHashResult = AstNormalizer.normalizeQuery(planContext, query, isCaseSensitive(), currentPlanHashMode); + // rewrite the query with operationDefinition RelationalLoggingUtil.publishNormalizeQueryLogs(message, stepTimeMicros(), astHashResult.getQueryCacheKey().getHash(), astHashResult.getQueryCacheKey().getCanonicalQueryString()); options = options.withChild(astHashResult.getQueryOptions()); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index 53f324bd70..cc1a4c472e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -391,6 +391,18 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi return ddlVisitor.visitIndexDefinition(ctx); } + @Nonnull + @Override + public Object visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { + return ddlVisitor.visitFunctionDefinition(ctx); + } + + @Nonnull + @Override + public Object visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { + return visitChildren(ctx); + } + @Override public Object visitIndexAttributes(RelationalParser.IndexAttributesContext ctx) { return visitChildren(ctx); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index dab0c52689..57d748f6bd 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -32,6 +32,7 @@ import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerUserDefinedFunction; import com.apple.foundationdb.relational.recordlayer.query.Identifier; import com.apple.foundationdb.relational.recordlayer.query.IndexGenerator; import com.apple.foundationdb.relational.recordlayer.query.LogicalOperator; @@ -174,6 +175,12 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi @Nonnull @Override + public RecordLayerUserDefinedFunction visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { + return new RecordLayerUserDefinedFunction(ctx.functionName.getText(), ctx.inputTypeName.getText(), ctx.columnType(), ctx.paramName.getText(), ctx.expression()); + } + + @Nonnull + @Override public DataType.Named visitEnumDefinition(@Nonnull RelationalParser.EnumDefinitionContext ctx) { final var enumId = visitUid(ctx.uid()); @@ -208,6 +215,7 @@ public ProceduralPlan visitCreateSchemaTemplateStatement(@Nonnull RelationalPars final ImmutableSet.Builder structClauses = ImmutableSet.builder(); final ImmutableSet.Builder tableClauses = ImmutableSet.builder(); final ImmutableSet.Builder indexClauses = ImmutableSet.builder(); + final ImmutableSet.Builder functionClauses = ImmutableSet.builder(); for (final var templateClause : ctx.templateClause()) { if (templateClause.enumDefinition() != null) { metadataBuilder.addAuxiliaryType(visitEnumDefinition(templateClause.enumDefinition())); @@ -215,6 +223,8 @@ public ProceduralPlan visitCreateSchemaTemplateStatement(@Nonnull RelationalPars structClauses.add(templateClause.structDefinition()); } else if (templateClause.tableDefinition() != null) { tableClauses.add(templateClause.tableDefinition()); + } else if (templateClause.functionDefinition() != null) { + functionClauses.add(templateClause.functionDefinition()); } else { Assert.thatUnchecked(templateClause.indexDefinition() != null); indexClauses.add(templateClause.indexDefinition()); @@ -228,6 +238,14 @@ public ProceduralPlan visitCreateSchemaTemplateStatement(@Nonnull RelationalPars final var tableWithIndex = RecordLayerTable.Builder.from(table).addIndex(index).build(); metadataBuilder.addTable(tableWithIndex); } + final var functions = functionClauses.build().stream().map(this::visitFunctionDefinition).collect(ImmutableList.toImmutableList()); + for (final var func : functions) { + // (TODO): maybe a table type too? + final var auxiliaryType = metadataBuilder.extractAuxiliaryType(func.getInputTypeName()); + System.out.println("auxiliaryType:" + auxiliaryType); + Assert.thatUnchecked(auxiliaryType instanceof DataType.StructType); + metadataBuilder.addAuxiliaryType(((DataType.StructType) auxiliaryType).withFunctionDefinition(func)); + } return ProceduralPlan.of(metadataOperationsFactory.getCreateSchemaTemplateConstantAction(metadataBuilder.build(), Options.NONE)); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index 4671921f8c..114832a184 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -229,6 +229,18 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi return getDelegate().visitIndexDefinition(ctx); } + @Nonnull + @Override + public Object visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { + return getDelegate().visitFunctionDefinition(ctx); + } + + @Nonnull + @Override + public Object visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { + return getDelegate().visitUserDefinedFunctionCall(ctx); + } + @Nonnull @Override public Object visitIndexAttributes(@Nonnull RelationalParser.IndexAttributesContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index 2468faa999..3f54ebad27 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -36,6 +36,8 @@ import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.api.metadata.DataType; +import com.apple.foundationdb.relational.api.metadata.FunctionDefinition; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; import com.apple.foundationdb.relational.recordlayer.query.Expression; @@ -181,6 +183,7 @@ public Expression visitGroupByItem(@Nonnull RelationalParser.GroupByItemContext @Nonnull @Override public Expression visitAggregateFunctionCall(@Nonnull RelationalParser.AggregateFunctionCallContext functionCon) { + System.out.println("visitAggregateFunctionCall called"); return visitAggregateWindowedFunction(functionCon.aggregateWindowedFunction()); } @@ -248,9 +251,41 @@ public Expression visitCaseFunctionCall(@Nonnull RelationalParser.CaseFunctionCa return Expression.ofUnnamed(new PickValue(new ConditionSelectorValue(implications.build()), pickerValues.build())); } + @Nonnull + @Override + public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { + System.out.println("visitUserDefinedFunctionCall called"); + final var functionName = ctx.userDefinedFunctionName.getText(); + // special case for user-defined functions where we want to exclude the first argument from + // being literal-stripped. + @Nonnull Expressions arguments; + boolean isUdf = getDelegate().getSemanticAnalyzer().isUdfFunction(functionName); + if (isUdf) { + final var argumentNodes = ctx.functionArgs().children.stream() + .filter(arg -> arg instanceof RelationalParser.FunctionArgContext) + .map(RelationalParser.FunctionArgContext.class::cast) + .collect(Collectors.toUnmodifiableList()); + Assert.thatUnchecked(!argumentNodes.isEmpty()); + final var classNameExpression = getDelegate().getPlanGenerationContext().withDisabledLiteralProcessing(() -> { + final var result = visitFunctionArg(argumentNodes.get(0)); + Assert.thatUnchecked(result.getUnderlying() instanceof LiteralValue, + ErrorCode.INVALID_ARGUMENT_FOR_FUNCTION, () -> String.format("attempt to invoke java_call with incorrect UDF '%s'", + result.getUnderlying())); + return result; + }); + arguments = Expressions.of(Streams.concat(Stream.of(classNameExpression), + argumentNodes.stream().skip(1).map(this::visitFunctionArg)) + .collect(Collectors.toUnmodifiableList())); + } else { + arguments = visitFunctionArgs(ctx.functionArgs()); + } + return getDelegate().resolveFunction(functionName, arguments.asList().toArray(new Expression[0])); + } + @Nonnull @Override public Expression visitFunctionCallExpressionAtom(@Nonnull RelationalParser.FunctionCallExpressionAtomContext ctx) { + System.out.println("ExpressionVisitor::visitFunctionCallExpressionAtom:" + ctx.getText() + " ctx class:" + ctx.getClass()); return parseChild(ctx); } @@ -465,8 +500,20 @@ public Expression visitBitExpressionAtom(@Nonnull RelationalParser.BitExpression @Nonnull @Override public Expression visitBinaryComparisonPredicate(@Nonnull RelationalParser.BinaryComparisonPredicateContext ctx) { + // if comparison operator is overriden, and left.dataType = RLUDOperator.getStructName, + // for example, ctx = (a = b) + // RLUDoperator.getExpressionCtx = (x.string_value = y.string_value and x.long_value = y.long_value); + // rewrite the function, change ctx -> a.string_value = b.string_value and a.long_value = b.long_value + System.out.println("get into visitBinaryComparisonPredicate"); final var left = Assert.castUnchecked(ctx.left.accept(this), Expression.class); final var right = Assert.castUnchecked(ctx.right.accept(this), Expression.class); + System.out.println("left dataType:" + left.getDataType() + " left:" + left + " left.getClass:" + left.getDataType().getClass()); + System.out.println("right dataType:" + right.getDataType()); + + if (left.getDataType() instanceof DataType.StructType) { + FunctionDefinition operationDefinition = getDelegate().getCatalog().getOperationDefinition(ctx.comparisonOperator().getText(), (DataType.StructType) left.getDataType()); + System.out.println("operationDefinition:" + operationDefinition); + } return getDelegate().resolveFunction(ctx.comparisonOperator().getText(), left, right); } @@ -822,6 +869,11 @@ Optional getStateMaybe() { @Nonnull private Expression parseChild(@Nonnull ParserRuleContext context) { + Expressions expressions = (Expressions) visitChildren(context); + for (int i = 0; i < context.getChildCount(); i++) { + System.out.println("ith:" + i + " child:" + context.getChild(i).getText()); + } + System.out.println("expressions:" + expressions); return Assert.castUnchecked(visitChildren(context), Expression.class); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/IdentifierVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/IdentifierVisitor.java index 73b0a44acb..4467109f9b 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/IdentifierVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/IdentifierVisitor.java @@ -54,6 +54,7 @@ public Identifier visitTableName(@Nonnull RelationalParser.TableNameContext tabl @Nonnull @Override public Identifier visitFullId(@Nonnull RelationalParser.FullIdContext fullIdContext) { + System.out.println("identifierVisitor visitFullId called"); Assert.thatUnchecked(!fullIdContext.uid().isEmpty()); final ImmutableList.Builder qualifierBuilder = ImmutableList.builder(); for (int i = 0; i < fullIdContext.uid().size() - 1; i++) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java index 55acc9f7c2..a12cbb0f18 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java @@ -164,6 +164,10 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefinitionContext ctx); + @Nonnull + @Override + Object visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx); + @Override Object visitIndexAttributes(RelationalParser.IndexAttributesContext ctx); diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java index 1e7a2dca9a..0493020192 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java @@ -30,6 +30,7 @@ import java.sql.DriverManager; import java.sql.SQLException; +import java.util.stream.Collectors; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @@ -44,8 +45,8 @@ class DriverManagerTest { @BeforeEach public void beforeEach() throws SQLException { // Cleanup old drivers, if any, - final var oldDrivers = DriverManager.drivers().collect(ImmutableList.toImmutableList()); - for (final var driver : oldDrivers) { + final var oldDrivers = DriverManager.drivers().collect(Collectors.toList()); + for (final var driver: oldDrivers) { DriverManager.deregisterDriver(driver); } DriverManager.drivers().forEach(driver -> { diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index 2cccbf96b3..82d4ec2c2e 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -876,6 +876,39 @@ void aliasingTableToResolveAmbiguityWorks() throws Exception { } } + @Test + void testUserDefinedFunction() throws Exception { + final String schemaTemplate = "CREATE TYPE AS STRUCT Location (name string, latitude string, longitude string)" + + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + + "CREATE FUNCTION lat(x Location) RETURNS (string) AS (x.latitude)\n"; + + try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { + try (var s = ddl.setSchemaAndGetConnection().createStatement()) { + insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); + } + try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE lat(loc) = ?loc")) { + ps.setString("loc", "37.3"); + try (final RelationalResultSet resultSet = ps.executeQuery()) { + ResultSetAssert.assertThat(resultSet).hasNextRow(); + } + } + } + } + + private RelationalStruct insertLocationComplexRecord(RelationalStatement s, long uid, @Nonnull final String name, String latitude, String longitude) throws SQLException { + var struct = EmbeddedRelationalStruct.newBuilder() + .addLong("UID", uid) + .addStruct("LOC", EmbeddedRelationalStruct.newBuilder() + .addString("NAME", name) + .addString("LATITUDE", latitude) + .addString("LONGITUDE", longitude) + .build()) + .build(); + int cnt = s.executeInsert("T1", struct); + Assertions.assertEquals(1, cnt, "Incorrect insertion count"); + return struct; + } + @Test void testBitmap() throws Exception { final String query = "SELECT BITMAP_CONSTRUCT_AGG(BITMAP_BIT_POSITION(uid)) as bitmap, category, BITMAP_BUCKET_OFFSET(uid) as offset FROM T1\n" + From 74602e70501e438cd07725ccf84a369bc46c38fc Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 5 Dec 2024 12:47:32 -0800 Subject: [PATCH 18/41] first clean --- .../relational/api/metadata/DataType.java | 22 +++---------------- ...RecordLayerStoreSchemaTemplateCatalog.java | 2 -- .../metadata/RecordLayerSchemaTemplate.java | 14 +++++++++++- .../recordlayer/query/PlanGenerator.java | 1 - .../query/functions/SqlFunctionCatalog.java | 6 +++++ .../query/visitors/ExpressionVisitor.java | 22 +++---------------- .../query/visitors/IdentifierVisitor.java | 1 - .../relational/api/DriverManagerTest.java | 1 - 8 files changed, 25 insertions(+), 44 deletions(-) diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java index cb95725d6e..59b8896909 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java @@ -957,17 +957,13 @@ public static final class StructType extends DataType implements Named { @Nonnull private final String name; - @Nonnull - private final Map userDefinedFunctions; - @Nonnull private final Supplier resolvedSupplier = Suppliers.memoize(this::calculateResolved); - private StructType(@Nonnull final String name, boolean isNullable, @Nonnull final List fields, @Nonnull final Map userDefinedOperators) { + private StructType(@Nonnull final String name, boolean isNullable, @Nonnull final List fields) { super(isNullable, false, Code.STRUCT); this.name = name; this.fields = ImmutableList.copyOf(fields); - this.userDefinedFunctions = ImmutableMap.copyOf(userDefinedOperators); } @Nonnull @@ -981,11 +977,6 @@ public String getName() { return name; } - @Nonnull - public Map getUserDefinedFunctions() { - return ImmutableMap.copyOf(userDefinedFunctions); - } - public static class Field { @Nonnull private final Supplier hashCodeSupplier = Suppliers.memoize(this::computeHashCode); @@ -1056,14 +1047,7 @@ public String toString() { @Nonnull public static StructType from(@Nonnull final String name, @Nonnull final List fields, boolean isNullable) { - return new StructType(name, isNullable, fields, ImmutableMap.of()); - } - - @Nonnull - public StructType withFunctionDefinition(FunctionDefinition functionDefinition) { - Map newUserDefinedFunctions = new java.util.HashMap<>(userDefinedFunctions); - newUserDefinedFunctions.put(functionDefinition.getName(), functionDefinition); - return new StructType(name, isNullable(), fields, newUserDefinedFunctions); + return new StructType(name, isNullable, fields); } @Override @@ -1072,7 +1056,7 @@ public StructType withNullable(boolean isNullable) { if (isNullable == isNullable()) { return this; } - return new StructType(name, isNullable, fields, userDefinedFunctions); + return new StructType(name, isNullable, fields); } private boolean calculateResolved() { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/RecordLayerStoreSchemaTemplateCatalog.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/RecordLayerStoreSchemaTemplateCatalog.java index 03c3e69a2e..95fb7b0434 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/RecordLayerStoreSchemaTemplateCatalog.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/catalog/RecordLayerStoreSchemaTemplateCatalog.java @@ -160,7 +160,6 @@ private static Tuple getSchemaTemplatePrimaryKey(String schemaTemplateName) { @Override public SchemaTemplate loadSchemaTemplate(@Nonnull Transaction txn, @Nonnull String templateName) throws RelationalException { - System.out.println("RLStoreSchemaTemplateCatalog loadSchemaTemplate called with templateName:" + templateName); Tuple key = getSchemaTemplatePrimaryKey(templateName); var recordStore = RecordLayerStoreUtils.openRecordStore(txn, this.catalogSchemaPath, this.catalogRecordMetaDataProvider); @@ -203,7 +202,6 @@ public SchemaTemplate loadSchemaTemplate(@Nonnull Transaction txn, @Nonnull Stri */ @Nonnull private static SchemaTemplate toSchemaTemplate(@Nonnull Message m) throws InvalidProtocolBufferException { - System.out.println("RecordLayerStoreSchemaTemplateCatalog::toSchemaTemplate called"); Descriptors.Descriptor d = m.getDescriptorForType(); ByteString bs = (ByteString) m.getField(d.findFieldByName(SchemaTemplateSystemTable.METADATA)); RecordMetaData metaData = RecordMetaData.build(RecordMetaDataProto.MetaData.parseFrom(bs.toByteArray())); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java index 48fd3d92f5..f48865d702 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java @@ -402,7 +402,6 @@ public Builder addAuxiliaryType(@Nonnull DataType.Named auxiliaryType) { Assert.thatUnchecked(!tables.containsKey(auxiliaryType.getName()), ErrorCode.INVALID_SCHEMA_TEMPLATE, TABLE_ALREADY_EXISTS, auxiliaryType.getName()); Assert.thatUnchecked(!auxiliaryTypes.containsKey(auxiliaryType.getName()), ErrorCode.INVALID_SCHEMA_TEMPLATE, TYPE_WITH_NAME_ALREADY_EXISTS, auxiliaryType.getName()); auxiliaryTypes.put(auxiliaryType.getName(), auxiliaryType); - System.out.println("auxiliaryTypes name:" + auxiliaryType.getName()); return this; } @@ -420,6 +419,19 @@ public Builder addAuxiliaryTypes(@Nonnull Collection auxiliaryTy return this; } + @Nonnull + public Builder addUDF(@Nonnull UDF udf) { + Assert.thatUnchecked(!udfMap.containsKey(udf.getUdfName()), ErrorCode.INVALID_SCHEMA_TEMPLATE, TABLE_ALREADY_EXISTS, udf.getUdfName()); + udfMap.put(udf.getUdfName(), udf); + return this; + } + + @Nonnull + public Builder addUDFs(@Nonnull Collection udf) { + udf.forEach(this::addUDF); + return this; + } + @Nonnull Builder setCachedMetadata(@Nonnull final RecordMetaData metadata) { this.cachedMetadata = metadata; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java index 9ca7efcf11..586508d718 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/PlanGenerator.java @@ -153,7 +153,6 @@ private Plan getPlanInternal(@Nonnull String query, @Nonnull KeyValueLogMessa QueryPlan.PhysicalQueryPlan.getValidPlanHashModes(options); final PlanHashable.PlanHashMode currentPlanHashMode = QueryPlan.PhysicalQueryPlan.getCurrentPlanHashMode(options); final var astHashResult = AstNormalizer.normalizeQuery(planContext, query, isCaseSensitive(), currentPlanHashMode); - // rewrite the query with operationDefinition RelationalLoggingUtil.publishNormalizeQueryLogs(message, stepTimeMicros(), astHashResult.getQueryCacheKey().getHash(), astHashResult.getQueryCacheKey().getCanonicalQueryString()); options = options.withChild(astHashResult.getQueryOptions()); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java index 1abfabda9c..959a9e500a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java @@ -115,6 +115,12 @@ public static SqlFunctionCatalog instance() { return INSTANCE; } + public BuiltInFunction addFunction(UserDefinedFunctionDefinition functionDefinition) { + final var func = functionDefinition.getBuiltInFunction(); + udfMap.put(functionDefinition.getName(), func); + return func; + } + /** * A utility method that transforms a single-item {@link RecordConstructorValue} value into its inner {@link Value}. * This is mainly used for deterministically distinguishing between: diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index 3f54ebad27..e4ea7bb3b9 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -254,11 +254,13 @@ public Expression visitCaseFunctionCall(@Nonnull RelationalParser.CaseFunctionCa @Nonnull @Override public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { - System.out.println("visitUserDefinedFunctionCall called"); final var functionName = ctx.userDefinedFunctionName.getText(); // special case for user-defined functions where we want to exclude the first argument from // being literal-stripped. @Nonnull Expressions arguments; + // check if this function exists in schema template + UDF udf = getDelegate().getCatalog().getUDF(functionName); + // look up in FunctionCatalog boolean isUdf = getDelegate().getSemanticAnalyzer().isUdfFunction(functionName); if (isUdf) { final var argumentNodes = ctx.functionArgs().children.stream() @@ -285,7 +287,6 @@ public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDef @Nonnull @Override public Expression visitFunctionCallExpressionAtom(@Nonnull RelationalParser.FunctionCallExpressionAtomContext ctx) { - System.out.println("ExpressionVisitor::visitFunctionCallExpressionAtom:" + ctx.getText() + " ctx class:" + ctx.getClass()); return parseChild(ctx); } @@ -500,20 +501,8 @@ public Expression visitBitExpressionAtom(@Nonnull RelationalParser.BitExpression @Nonnull @Override public Expression visitBinaryComparisonPredicate(@Nonnull RelationalParser.BinaryComparisonPredicateContext ctx) { - // if comparison operator is overriden, and left.dataType = RLUDOperator.getStructName, - // for example, ctx = (a = b) - // RLUDoperator.getExpressionCtx = (x.string_value = y.string_value and x.long_value = y.long_value); - // rewrite the function, change ctx -> a.string_value = b.string_value and a.long_value = b.long_value - System.out.println("get into visitBinaryComparisonPredicate"); final var left = Assert.castUnchecked(ctx.left.accept(this), Expression.class); final var right = Assert.castUnchecked(ctx.right.accept(this), Expression.class); - System.out.println("left dataType:" + left.getDataType() + " left:" + left + " left.getClass:" + left.getDataType().getClass()); - System.out.println("right dataType:" + right.getDataType()); - - if (left.getDataType() instanceof DataType.StructType) { - FunctionDefinition operationDefinition = getDelegate().getCatalog().getOperationDefinition(ctx.comparisonOperator().getText(), (DataType.StructType) left.getDataType()); - System.out.println("operationDefinition:" + operationDefinition); - } return getDelegate().resolveFunction(ctx.comparisonOperator().getText(), left, right); } @@ -869,11 +858,6 @@ Optional getStateMaybe() { @Nonnull private Expression parseChild(@Nonnull ParserRuleContext context) { - Expressions expressions = (Expressions) visitChildren(context); - for (int i = 0; i < context.getChildCount(); i++) { - System.out.println("ith:" + i + " child:" + context.getChild(i).getText()); - } - System.out.println("expressions:" + expressions); return Assert.castUnchecked(visitChildren(context), Expression.class); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/IdentifierVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/IdentifierVisitor.java index 4467109f9b..73b0a44acb 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/IdentifierVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/IdentifierVisitor.java @@ -54,7 +54,6 @@ public Identifier visitTableName(@Nonnull RelationalParser.TableNameContext tabl @Nonnull @Override public Identifier visitFullId(@Nonnull RelationalParser.FullIdContext fullIdContext) { - System.out.println("identifierVisitor visitFullId called"); Assert.thatUnchecked(!fullIdContext.uid().isEmpty()); final ImmutableList.Builder qualifierBuilder = ImmutableList.builder(); for (int i = 0; i < fullIdContext.uid().size() - 1; i++) { diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java index 0493020192..5948d12ea7 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java @@ -30,7 +30,6 @@ import java.sql.DriverManager; import java.sql.SQLException; -import java.util.stream.Collectors; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; From fa28a85817dbb5e8c06c7641503dc7a4571f5cea Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 11 Dec 2024 13:35:57 -0800 Subject: [PATCH 19/41] save --- .../relational/api/metadata/DataType.java | 1 - .../api/metadata/FunctionDefinition.java | 30 ------- .../src/main/antlr/RelationalParser.g4 | 2 +- .../metadata/RecordLayerSchemaTemplate.java | 1 + .../UserDefinedFunctionDefinition.java | 81 +++++++++++++++++++ .../recordlayer/query/IndexGenerator.java | 6 +- .../recordlayer/query/SemanticAnalyzer.java | 69 ++++++++++++++-- .../query/visitors/BaseVisitor.java | 1 + .../query/visitors/DdlVisitor.java | 56 +++++++++++-- .../recordlayer/query/StandardQueryTests.java | 2 +- 10 files changed, 203 insertions(+), 46 deletions(-) delete mode 100644 fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/FunctionDefinition.java create mode 100644 fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java index 59b8896909..96ed59a281 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/DataType.java @@ -29,7 +29,6 @@ import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import javax.annotation.Nonnull; import java.sql.Types; diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/FunctionDefinition.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/FunctionDefinition.java deleted file mode 100644 index d42568f484..0000000000 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/metadata/FunctionDefinition.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * FunctionDefinition.java - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apple.foundationdb.relational.api.metadata; - -import javax.annotation.Nonnull; - -public interface FunctionDefinition extends Metadata{ - @Override - default void accept(@Nonnull final Visitor visitor) { - visitor.visit(this); - } -} diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index c625e3a1d2..b13bf415ad 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -155,7 +155,7 @@ indexDefinition ; functionDefinition - : FUNCTION functionName=uid LEFT_ROUND_BRACKET paramName=uid inputTypeName=uid RIGHT_ROUND_BRACKET RETURNS LEFT_ROUND_BRACKET columnType ( ',' columnType)* RIGHT_ROUND_BRACKET AS LEFT_ROUND_BRACKET expression ( ',' expression)* RIGHT_ROUND_BRACKET + : FUNCTION functionName=uid LEFT_ROUND_BRACKET paramName=uid inputTypeName=columnType RIGHT_ROUND_BRACKET RETURNS columnType AS fullId ; indexAttributes diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java index f48865d702..e8f3ea5dfd 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java @@ -215,6 +215,7 @@ public static RecordLayerSchemaTemplate fromRecordMetadataWithFakeTemplateNameAn @Override public Optional findTableByName(@Nonnull final String tableName) { for (final var table : getTables()) { + System.out.println("findTableByName:" + table.getName()); if (table.getName().equals(tableName)) { return Optional.of(table); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java new file mode 100644 index 0000000000..b6d93eb6cf --- /dev/null +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java @@ -0,0 +1,81 @@ +/* + * UserDefinedFunctionDefinition.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.relational.recordlayer.metadata; + +import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; +import com.apple.foundationdb.record.metadata.expressions.KeyExpression; +import com.apple.foundationdb.record.metadata.UDF; +import com.apple.foundationdb.record.metadata.expressions.NestingKeyExpression; +import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; +import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Locale; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class UserDefinedFunctionDefinition { + @Nonnull private final String functionName; + @Nonnull private final KeyExpression keyExpression; + + public UserDefinedFunctionDefinition(@Nonnull String functionName, @Nonnull KeyExpression keyExpression) { + this.functionName = functionName; + this.keyExpression = keyExpression; + } + + @Nonnull + public String getName() { + return functionName; + } + + @Nonnull + public KeyExpression getKeyExpression() {return keyExpression;} + + public BuiltInFunction getBuiltInFunction() { + return new BuiltInFunction(functionName, List.of(), (builtInFunction, arguments) -> FieldValue.ofFieldNames((Value) arguments.get(0), getFieldNamesFromKeyExpression(keyExpression))) { + @Nonnull + @Override + public String getFunctionName() { + return functionName; + } + }; + } + + private List getFieldNamesFromKeyExpression(KeyExpression keyExpression) { + if (keyExpression instanceof NestingKeyExpression) { + return Stream.concat(List.of(((NestingKeyExpression) keyExpression).getParent().getFieldName().toUpperCase(Locale.ROOT)).stream(), getFieldNamesFromKeyExpression(((NestingKeyExpression) keyExpression).getChild()).stream()).collect(Collectors.toList()); + } else { + return List.of(((FieldKeyExpression) keyExpression).getFieldName().toUpperCase(Locale.ROOT)); + } + } + + public UDF toUDF() { + return new UDF(functionName, keyExpression); + } + + public static UserDefinedFunctionDefinition fromUDF(UDF udf) { + return new UserDefinedFunctionDefinition(udf.getUdfName(), udf.getKeyExpression()); + } +} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/IndexGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/IndexGenerator.java index 52e93e663d..c2b412d885 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/IndexGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/IndexGenerator.java @@ -517,7 +517,7 @@ private KeyExpression toKeyExpression(Value value, Map orderingFu } @Nonnull - private KeyExpression toKeyExpression(@Nonnull Value value) { + public static KeyExpression toKeyExpression(@Nonnull Value value) { if (value instanceof VersionValue) { return VersionKeyExpression.VERSION; } else if (value instanceof FieldValue) { @@ -732,12 +732,12 @@ private Value dereference(@Nonnull Value value) { } @Nonnull - private KeyExpression toKeyExpression(@Nonnull List> fields) { + private static KeyExpression toKeyExpression(@Nonnull List> fields) { return toKeyExpression(fields, 0); } @Nonnull - private KeyExpression toKeyExpression(@Nonnull List> fields, int index) { + private static KeyExpression toKeyExpression(@Nonnull List> fields, int index) { Assert.thatUnchecked(!fields.isEmpty()); final var field = fields.get(index); final var keyExpression = toKeyExpression(field.getLeft(), field.getRight()); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index 10d97d5394..6caae92622 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -39,6 +39,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.JavaCallFunction; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; import com.apple.foundationdb.record.query.plan.cascades.values.NotValue; +import com.apple.foundationdb.record.query.plan.cascades.values.ObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.RelOpValue; import com.apple.foundationdb.record.query.plan.cascades.values.StreamableAggregateValue; @@ -373,7 +374,7 @@ private List lookup(@Nonnull Identifier referenceIdentifier, continue; } } - final var nestedFieldMaybe = lookupNestedField(referenceIdentifier, attribute, operator, matchQualifiedOnly); + final var nestedFieldMaybe = resolveIdentifierInType(referenceIdentifier, attribute, operator, matchQualifiedOnly); if (nestedFieldMaybe.isPresent()) { matchedAttributes.add(nestedFieldMaybe.get()); checkForPseudoColumns = false; @@ -413,10 +414,10 @@ public Optional lookupAlias(@Nonnull Identifier requestedAlias, } @Nonnull - public Optional lookupNestedField(@Nonnull Identifier requestedIdentifier, - @Nonnull Expression existingExpression, - @Nonnull LogicalOperator logicalOperator, - boolean matchQualifiedOnly) { + public Optional resolveIdentifierInType(@Nonnull Identifier requestedIdentifier, + @Nonnull Expression existingExpression, + @Nonnull LogicalOperator logicalOperator, + boolean matchQualifiedOnly) { if (existingExpression.getName().isEmpty() || requestedIdentifier.fullyQualifiedName().size() <= 1) { return Optional.empty(); } @@ -467,6 +468,64 @@ public Optional lookupNestedField(@Nonnull Identifier requestedIdent return Optional.of(nestedAttribute); } + @Nonnull + public Optional resolveIdentifierInType(@Nonnull Identifier requestedIdentifier, + @Nonnull String param, + @Nonnull DataType existingDataType) { + /* + if (existingExpression.getName().isEmpty() || requestedIdentifier.fullyQualifiedName().size() <= 1) { + return Optional.empty(); + } + final var effectiveExistingExpr = matchQualifiedOnly && logicalOperator.getName().isPresent() ? + existingExpression.withQualifier(Optional.of(logicalOperator.getName().get())) : + existingExpression.clearQualifier(); + var effectiveExprName = effectiveExistingExpr.getName().orElseThrow(); + + if (!requestedIdentifier.prefixedWith(effectiveExprName)) { + if (existingExpression.getName().isPresent() && + requestedIdentifier.prefixedWith(existingExpression.getName().get())) { + effectiveExprName = existingExpression.getName().get(); + } else { + return Optional.empty(); + } + } + */ + // requestedIdentifier = x.latitude + ObjectValue objectValue = ObjectValue.of(CorrelationIdentifier.of("PARAM"), DataTypeUtils.toRecordLayerType(existingDataType)); + final var remainingPath = requestedIdentifier.fullyQualifiedName(); + /* + if (remainingPath.isEmpty()) { + return Optional.of(existingExpression.withName(requestedIdentifier)); + } + + */ + final ImmutableList.Builder accessors = ImmutableList.builder(); + DataType currentDataType = existingDataType; + for (String s : remainingPath) { + if (currentDataType.getCode() != DataType.Code.STRUCT) { + return Optional.empty(); + } + final var fields = ((DataType.StructType) currentDataType).getFields(); + var found = false; + for (int j = 0; j < fields.size(); j++) { + if (fields.get(j).getName().equals(s)) { + accessors.add(new FieldValue.Accessor(fields.get(j).getName(), j)); + currentDataType = fields.get(j).getType(); + found = true; + break; + } + } + if (!found) { + return Optional.empty(); + } + } + // probably need to check if currentDataType = targetDataType + final var fieldPath = FieldValue.resolveFieldPath(DataTypeUtils.toRecordLayerType(existingDataType), accessors.build()); + final var attributeExpression = FieldValue.ofFieldsAndFuseIfPossible(objectValue, fieldPath); + final var nestedAttribute = new Expression(Optional.of(requestedIdentifier), existingDataType, attributeExpression); + return Optional.of(nestedAttribute); + } + @Nonnull public DataType lookupType(@Nonnull Identifier typeIdentifier, boolean isNullable, boolean isRepeated, @Nonnull Function> dataTypeProvider) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index cc1a4c472e..e38274c260 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -180,6 +180,7 @@ boolean isTopLevel() { @Nonnull LogicalPlanFragment pushPlanFragment() { + System.out.println("pushPlanFragment is called"); currentPlanFragment = Optional.of(currentPlanFragment.map(LogicalPlanFragment::addChild).orElse(LogicalPlanFragment.ofRoot())); return currentPlanFragment.get(); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 57d748f6bd..cd3842e774 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -21,7 +21,7 @@ package com.apple.foundationdb.relational.recordlayer.query.visitors; import com.apple.foundationdb.annotation.API; - +import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalSortExpression; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.ddl.MetadataOperationsFactory; @@ -32,7 +32,8 @@ import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; -import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerUserDefinedFunction; +import com.apple.foundationdb.relational.recordlayer.metadata.UserDefinedFunctionDefinition; +import com.apple.foundationdb.relational.recordlayer.query.Expression; import com.apple.foundationdb.relational.recordlayer.query.Identifier; import com.apple.foundationdb.relational.recordlayer.query.IndexGenerator; import com.apple.foundationdb.relational.recordlayer.query.LogicalOperator; @@ -46,6 +47,7 @@ import javax.annotation.Nonnull; import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -175,11 +177,30 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi @Nonnull @Override - public RecordLayerUserDefinedFunction visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { - return new RecordLayerUserDefinedFunction(ctx.functionName.getText(), ctx.inputTypeName.getText(), ctx.columnType(), ctx.paramName.getText(), ctx.expression()); + public UserDefinedFunctionDefinition visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { + final var id = visitFullId(ctx.fullId()); + // (TODO) remove the input param prefix from id + Assert.thatUnchecked(id.prefixedWith(Identifier.of(ctx.paramName.getText())), "Invalid function definition"); + Identifier newId = Identifier.of("LATITUDE"); + + final var ddlCatalog = metadataBuilder.build(); + // parse the function definition using the newly constructed metadata. + getDelegate().replaceCatalog(ddlCatalog); + final var semanticAnalyzer = getDelegate().getSemanticAnalyzer(); + // input column type + final var inputcolumnTypeId = ctx.columnType(0).customType != null ? visitUid(ctx.columnType(0).customType) : Identifier.of(ctx.columnType(0).getText()); + final var columnType = semanticAnalyzer.lookupType(inputcolumnTypeId, true, false, metadataBuilder::findType); + // (TODO) throw ex if not found + + // look up the identifier in a type + // for example, inputParamType = Person(string name, Location address) Location(string street, string zipcode) + // resolveIdentifierInType("street", Person) -> Expression("name"="street", dataType=Person, underlying=FieldValue(PARAM.address.street) -> keyExpression = field("address").nest("street")) + Expression expression = semanticAnalyzer.resolveIdentifierInType(newId, ctx.paramName.getText(), columnType).get(); + // (TODO) throw ex if empty + return new UserDefinedFunctionDefinition(ctx.functionName.getText(), IndexGenerator.toKeyExpression(expression.getUnderlying())); } - @Nonnull + @Nonnull @Override public DataType.Named visitEnumDefinition(@Nonnull RelationalParser.EnumDefinitionContext ctx) { final var enumId = visitUid(ctx.uid()); @@ -300,4 +321,29 @@ public ProceduralPlan visitDropSchemaTemplateStatement(@Nonnull RelationalParser public Boolean visitNullColumnConstraint(@Nonnull RelationalParser.NullColumnConstraintContext ctx) { return ctx.nullNotnull().NOT() == null; } + + private static RecordMetaDataProto.KeyExpression functionDefinitionToKeyExpression(String paramName, RelationalParser.ExpressionContext expressionContext) { + String[] fieldPath = expressionContext.getText().split("\\."); + Assert.thatUnchecked(fieldPath[0].equals(paramName), "Invalid function definition"); + fieldPath[0] = "PARAM"; + return arrayToKeyExpression(fieldPath); + } + + @Nonnull + private static RecordMetaDataProto.KeyExpression arrayToKeyExpression(@Nonnull String[] nameArray) { + RecordMetaDataProto.KeyExpression.Builder builder = RecordMetaDataProto.KeyExpression.newBuilder(); + if (nameArray.length == 1) { + return builder.setField(RecordMetaDataProto.Field.newBuilder() + .setFieldName(nameArray[0]) + .setFanType(RecordMetaDataProto.Field.FanType.SCALAR)) + .build(); + } else { + return builder.setNesting(RecordMetaDataProto.Nesting.newBuilder() + .setParent(RecordMetaDataProto.Field.newBuilder() + .setFieldName(nameArray[0]) + .setFanType(RecordMetaDataProto.Field.FanType.SCALAR)) + .setChild(arrayToKeyExpression(Arrays.copyOfRange(nameArray, 1, nameArray.length)))) + .build(); + } + } } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index 82d4ec2c2e..e6716f83f8 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -880,7 +880,7 @@ void aliasingTableToResolveAmbiguityWorks() throws Exception { void testUserDefinedFunction() throws Exception { final String schemaTemplate = "CREATE TYPE AS STRUCT Location (name string, latitude string, longitude string)" + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + - "CREATE FUNCTION lat(x Location) RETURNS (string) AS (x.latitude)\n"; + "CREATE FUNCTION lat(x Location) RETURNS string AS x.latitude\n"; try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { try (var s = ddl.setSchemaAndGetConnection().createStatement()) { From b0e5b37b6ddb8f0b583269340a9d38c8eb00f799 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Wed, 11 Dec 2024 20:24:42 -0800 Subject: [PATCH 20/41] store value --- .../UserDefinedFunctionDefinition.java | 38 +++------- .../recordlayer/query/Identifier.java | 7 ++ .../recordlayer/query/SemanticAnalyzer.java | 39 +++-------- .../query/visitors/DdlVisitor.java | 20 +++--- .../src/main/proto/continuation.proto | 1 + .../recordlayer/query/StandardQueryTests.java | 70 +++++++++++++++++-- 6 files changed, 101 insertions(+), 74 deletions(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java index b6d93eb6cf..f2c6233e76 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java @@ -20,29 +20,23 @@ package com.apple.foundationdb.relational.recordlayer.metadata; -import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; -import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.metadata.UDF; -import com.apple.foundationdb.record.metadata.expressions.NestingKeyExpression; import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.relational.util.Assert; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.util.List; -import java.util.Locale; -import java.util.stream.Collectors; -import java.util.stream.Stream; public class UserDefinedFunctionDefinition { @Nonnull private final String functionName; - @Nonnull private final KeyExpression keyExpression; + @Nonnull private final Value value; - public UserDefinedFunctionDefinition(@Nonnull String functionName, @Nonnull KeyExpression keyExpression) { + public UserDefinedFunctionDefinition(@Nonnull String functionName, @Nonnull Value value) { this.functionName = functionName; - this.keyExpression = keyExpression; + this.value = value; } @Nonnull @@ -50,32 +44,16 @@ public String getName() { return functionName; } - @Nonnull - public KeyExpression getKeyExpression() {return keyExpression;} - public BuiltInFunction getBuiltInFunction() { - return new BuiltInFunction(functionName, List.of(), (builtInFunction, arguments) -> FieldValue.ofFieldNames((Value) arguments.get(0), getFieldNamesFromKeyExpression(keyExpression))) { - @Nonnull - @Override - public String getFunctionName() { - return functionName; - } - }; - } - - private List getFieldNamesFromKeyExpression(KeyExpression keyExpression) { - if (keyExpression instanceof NestingKeyExpression) { - return Stream.concat(List.of(((NestingKeyExpression) keyExpression).getParent().getFieldName().toUpperCase(Locale.ROOT)).stream(), getFieldNamesFromKeyExpression(((NestingKeyExpression) keyExpression).getChild()).stream()).collect(Collectors.toList()); - } else { - return List.of(((FieldKeyExpression) keyExpression).getFieldName().toUpperCase(Locale.ROOT)); - } + Assert.thatUnchecked(value instanceof FieldValue, "Invalid UDF definition."); + return new BuiltInFunction(functionName, List.of(), (builtInFunction, arguments) -> FieldValue.ofFieldNames((Value) arguments.get(0), ((FieldValue) value).getFieldPathNames())) {}; } public UDF toUDF() { - return new UDF(functionName, keyExpression); + return new UDF(functionName, value); } public static UserDefinedFunctionDefinition fromUDF(UDF udf) { - return new UserDefinedFunctionDefinition(udf.getUdfName(), udf.getKeyExpression()); + return new UserDefinedFunctionDefinition(udf.getUdfName(), udf.getValue()); } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java index 0987c0394d..26b9c21347 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java @@ -128,6 +128,13 @@ public boolean prefixedWith(@Nonnull Identifier identifier) { return true; } + @Nonnull + public Identifier removePrefix(@Nonnull Identifier identifier) { + final var fullName = fullyQualifiedName(); + final var newIdFullName = fullName.subList(identifier.fullyQualifiedName().size(), fullName.size()); + return Identifier.of(newIdFullName.get(newIdFullName.size() - 1), newIdFullName.subList(0, newIdFullName.size() - 1)); + } + public boolean qualifiedWith(@Nonnull Identifier identifier) { final var identifierFullName = identifier.fullyQualifiedName(); final var fullName = fullyQualifiedName(); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index 6caae92622..db35de1d0a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -469,36 +469,18 @@ public Optional resolveIdentifierInType(@Nonnull Identifier requeste } @Nonnull - public Optional resolveIdentifierInType(@Nonnull Identifier requestedIdentifier, - @Nonnull String param, + public OptionalresolveIdentifierInType(@Nonnull Identifier requestedIdentifier, + @Nonnull Identifier paramId, @Nonnull DataType existingDataType) { - /* - if (existingExpression.getName().isEmpty() || requestedIdentifier.fullyQualifiedName().size() <= 1) { - return Optional.empty(); - } - final var effectiveExistingExpr = matchQualifiedOnly && logicalOperator.getName().isPresent() ? - existingExpression.withQualifier(Optional.of(logicalOperator.getName().get())) : - existingExpression.clearQualifier(); - var effectiveExprName = effectiveExistingExpr.getName().orElseThrow(); - - if (!requestedIdentifier.prefixedWith(effectiveExprName)) { - if (existingExpression.getName().isPresent() && - requestedIdentifier.prefixedWith(existingExpression.getName().get())) { - effectiveExprName = existingExpression.getName().get(); - } else { - return Optional.empty(); - } - } - */ - // requestedIdentifier = x.latitude + // requestedIdentifier = latitude + // need a value that represents a type? or make PFieldPath one of PValue? + Assert.thatUnchecked(requestedIdentifier.prefixedWith(paramId), "Invalid function definition"); ObjectValue objectValue = ObjectValue.of(CorrelationIdentifier.of("PARAM"), DataTypeUtils.toRecordLayerType(existingDataType)); - final var remainingPath = requestedIdentifier.fullyQualifiedName(); - /* - if (remainingPath.isEmpty()) { - return Optional.of(existingExpression.withName(requestedIdentifier)); - } - */ + if (requestedIdentifier.fullyQualifiedName().size() == paramId.fullyQualifiedName().size()) { + return Optional.of(FieldValue.ofFieldsAndFuseIfPossible(objectValue, FieldValue.FieldPath.empty())); + } + final var remainingPath = requestedIdentifier.removePrefix(paramId).fullyQualifiedName(); final ImmutableList.Builder accessors = ImmutableList.builder(); DataType currentDataType = existingDataType; for (String s : remainingPath) { @@ -522,8 +504,7 @@ public Optional resolveIdentifierInType(@Nonnull Identifier requeste // probably need to check if currentDataType = targetDataType final var fieldPath = FieldValue.resolveFieldPath(DataTypeUtils.toRecordLayerType(existingDataType), accessors.build()); final var attributeExpression = FieldValue.ofFieldsAndFuseIfPossible(objectValue, fieldPath); - final var nestedAttribute = new Expression(Optional.of(requestedIdentifier), existingDataType, attributeExpression); - return Optional.of(nestedAttribute); + return Optional.of(attributeExpression); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index cd3842e774..7b9591eacd 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalSortExpression; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.ddl.MetadataOperationsFactory; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; @@ -33,7 +34,6 @@ import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.recordlayer.metadata.UserDefinedFunctionDefinition; -import com.apple.foundationdb.relational.recordlayer.query.Expression; import com.apple.foundationdb.relational.recordlayer.query.Identifier; import com.apple.foundationdb.relational.recordlayer.query.IndexGenerator; import com.apple.foundationdb.relational.recordlayer.query.LogicalOperator; @@ -49,6 +49,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Optional; @API(API.Status.EXPERIMENTAL) @@ -178,26 +179,27 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi @Nonnull @Override public UserDefinedFunctionDefinition visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { + // id = x.latitude final var id = visitFullId(ctx.fullId()); - // (TODO) remove the input param prefix from id - Assert.thatUnchecked(id.prefixedWith(Identifier.of(ctx.paramName.getText())), "Invalid function definition"); - Identifier newId = Identifier.of("LATITUDE"); + final var paramNameId = Identifier.of(ctx.paramName.getText().toUpperCase(Locale.ROOT)); + // (TODO) if function = return x + // newId = latitude, x is removed + // final var newId = id.removePrefix(paramNameId); final var ddlCatalog = metadataBuilder.build(); // parse the function definition using the newly constructed metadata. getDelegate().replaceCatalog(ddlCatalog); final var semanticAnalyzer = getDelegate().getSemanticAnalyzer(); - // input column type + // columnType = DataType Location final var inputcolumnTypeId = ctx.columnType(0).customType != null ? visitUid(ctx.columnType(0).customType) : Identifier.of(ctx.columnType(0).getText()); final var columnType = semanticAnalyzer.lookupType(inputcolumnTypeId, true, false, metadataBuilder::findType); - // (TODO) throw ex if not found // look up the identifier in a type // for example, inputParamType = Person(string name, Location address) Location(string street, string zipcode) // resolveIdentifierInType("street", Person) -> Expression("name"="street", dataType=Person, underlying=FieldValue(PARAM.address.street) -> keyExpression = field("address").nest("street")) - Expression expression = semanticAnalyzer.resolveIdentifierInType(newId, ctx.paramName.getText(), columnType).get(); - // (TODO) throw ex if empty - return new UserDefinedFunctionDefinition(ctx.functionName.getText(), IndexGenerator.toKeyExpression(expression.getUnderlying())); + Optional expression = semanticAnalyzer.resolveIdentifierInType(id, paramNameId, columnType); + Assert.thatUnchecked(expression.isPresent(), "couldn't resolve function definition"); + return new UserDefinedFunctionDefinition(ctx.functionName.getText(), expression.get()); } @Nonnull diff --git a/fdb-relational-core/src/main/proto/continuation.proto b/fdb-relational-core/src/main/proto/continuation.proto index 4b5dfae0d3..595140cae6 100644 --- a/fdb-relational-core/src/main/proto/continuation.proto +++ b/fdb-relational-core/src/main/proto/continuation.proto @@ -21,6 +21,7 @@ syntax = "proto3"; package com.apple.foundationdb.relational.continuation; import "record_metadata.proto"; +import "record_value.proto"; import "record_query_plan.proto"; import "record_query_runtime.proto"; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index e6716f83f8..82a335ce39 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -876,15 +876,43 @@ void aliasingTableToResolveAmbiguityWorks() throws Exception { } } + // (TODO): more tests: 1) invalid param type lat(x loc) 2) return x @Test void testUserDefinedFunction() throws Exception { - final String schemaTemplate = "CREATE TYPE AS STRUCT Location (name string, latitude string, longitude string)" + + final String schemaTemplate = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + - "CREATE FUNCTION lat(x Location) RETURNS string AS x.latitude\n"; + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n" + + "CREATE FUNCTION name(x Location) RETURNS string AS x.name\n" + + "CREATE FUNCTION latlon(x LATLON) RETURNS LATLON AS x"; try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { try (var s = ddl.setSchemaAndGetConnection().createStatement()) { - insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); + insertLocationComplexRecord2(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); + } + // (TODO): double quote around loc, to differentiate udf from scalar function call, better way of doing it? + try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE name((loc)) = ?name")) { + ps.setString("name", "Apple Park Visitor Center"); + try (final RelationalResultSet resultSet = ps.executeQuery()) { + ResultSetAssert.assertThat(resultSet) + .hasNextRow() + .hasColumn("UID", 1L) + .hasNoNextRow(); + } + } + } + } + + @Test + void testUserDefinedFunction2() throws Exception { + final String schemaTemplate = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n"; + + try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { + try (var s = ddl.setSchemaAndGetConnection().createStatement()) { + insertLocationComplexRecord2(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); } try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE lat(loc) = ?loc")) { ps.setString("loc", "37.3"); @@ -895,13 +923,43 @@ void testUserDefinedFunction() throws Exception { } } - private RelationalStruct insertLocationComplexRecord(RelationalStatement s, long uid, @Nonnull final String name, String latitude, String longitude) throws SQLException { + // (TODO) not working yet, can fieldPath be empty? + @Test + void testUserDefinedFunction3() throws Exception { + final String schemaTemplate = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n" + + "CREATE FUNCTION name(x Location) RETURNS string AS x.name\n" + + "CREATE FUNCTION id(x uid) RETURNS bigint AS x"; + + try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { + try (var s = ddl.setSchemaAndGetConnection().createStatement()) { + insertLocationComplexRecord2(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); + } + // (TODO): double quote around loc, to differentiate udf from scalar function call, better way of doing it? + try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE id((uid)) = ?id")) { + ps.setLong("id", 1L); + try (final RelationalResultSet resultSet = ps.executeQuery()) { + ResultSetAssert.assertThat(resultSet) + .hasNextRow() + .hasColumn("UID", 1L) + .hasNoNextRow(); + } + } + } + } + + private RelationalStruct insertLocationComplexRecord2(RelationalStatement s, long uid, @Nonnull final String name, String latitude, String longitude) throws SQLException { + var coord = EmbeddedRelationalStruct.newBuilder() + .addString("LATITUDE", latitude) + .addString("LONGITUDE", longitude) + .build(); var struct = EmbeddedRelationalStruct.newBuilder() .addLong("UID", uid) .addStruct("LOC", EmbeddedRelationalStruct.newBuilder() .addString("NAME", name) - .addString("LATITUDE", latitude) - .addString("LONGITUDE", longitude) + .addStruct("COORD", coord) .build()) .build(); int cnt = s.executeInsert("T1", struct); From 91322a26b78cccf9715a6985b4972ef62219ef49 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 12 Dec 2024 14:07:15 -0800 Subject: [PATCH 21/41] project work --- .../UserDefinedFunctionDefinition.java | 25 +++++++++++++------ .../recordlayer/query/SemanticAnalyzer.java | 15 +++++------ .../query/visitors/DdlVisitor.java | 18 ++++++------- .../recordlayer/query/StandardQueryTests.java | 3 +-- 4 files changed, 33 insertions(+), 28 deletions(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java index f2c6233e76..286d3e4a07 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java @@ -23,9 +23,8 @@ import com.apple.foundationdb.record.metadata.UDF; import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; -import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; +import com.apple.foundationdb.record.query.plan.cascades.values.ObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; -import com.apple.foundationdb.relational.util.Assert; import javax.annotation.Nonnull; import java.util.List; @@ -33,10 +32,12 @@ public class UserDefinedFunctionDefinition { @Nonnull private final String functionName; @Nonnull private final Value value; + @Nonnull private final ObjectValue argumentValue; - public UserDefinedFunctionDefinition(@Nonnull String functionName, @Nonnull Value value) { + public UserDefinedFunctionDefinition(@Nonnull String functionName, @Nonnull Value value, @Nonnull ObjectValue argumentValue) { this.functionName = functionName; this.value = value; + this.argumentValue = argumentValue; } @Nonnull @@ -45,15 +46,25 @@ public String getName() { } public BuiltInFunction getBuiltInFunction() { - Assert.thatUnchecked(value instanceof FieldValue, "Invalid UDF definition."); - return new BuiltInFunction(functionName, List.of(), (builtInFunction, arguments) -> FieldValue.ofFieldNames((Value) arguments.get(0), ((FieldValue) value).getFieldPathNames())) {}; + return new BuiltInFunction(functionName, List.of(), (builtInFunction, arguments) -> replaceArgument((Value) arguments.get(0))) {}; } public UDF toUDF() { - return new UDF(functionName, value); + return new UDF(functionName, value, argumentValue); } public static UserDefinedFunctionDefinition fromUDF(UDF udf) { - return new UserDefinedFunctionDefinition(udf.getUdfName(), udf.getValue()); + return new UserDefinedFunctionDefinition(udf.getUdfName(), udf.getValue(), (ObjectValue) udf.getArgumentValue()); + } + + // replace the argumentValue with argument in value + private Value replaceArgument(Value argument) { + return value.replace((v) -> { + if (v instanceof ObjectValue && v.getResultType().equals(argumentValue.getResultType())) { + return argument; + } else { + return v; + } + }); } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index db35de1d0a..8e2bf5b774 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -469,20 +469,17 @@ public Optional resolveIdentifierInType(@Nonnull Identifier requeste } @Nonnull - public OptionalresolveIdentifierInType(@Nonnull Identifier requestedIdentifier, - @Nonnull Identifier paramId, - @Nonnull DataType existingDataType) { - // requestedIdentifier = latitude - // need a value that represents a type? or make PFieldPath one of PValue? + public Optional resolveIdentifierInType(@Nonnull Identifier requestedIdentifier, + @Nonnull Identifier paramId, + @Nonnull ObjectValue objectValue) { Assert.thatUnchecked(requestedIdentifier.prefixedWith(paramId), "Invalid function definition"); - ObjectValue objectValue = ObjectValue.of(CorrelationIdentifier.of("PARAM"), DataTypeUtils.toRecordLayerType(existingDataType)); if (requestedIdentifier.fullyQualifiedName().size() == paramId.fullyQualifiedName().size()) { - return Optional.of(FieldValue.ofFieldsAndFuseIfPossible(objectValue, FieldValue.FieldPath.empty())); + return Optional.of(objectValue); } final var remainingPath = requestedIdentifier.removePrefix(paramId).fullyQualifiedName(); final ImmutableList.Builder accessors = ImmutableList.builder(); - DataType currentDataType = existingDataType; + DataType currentDataType = DataTypeUtils.toRelationalType(objectValue.getResultType()); for (String s : remainingPath) { if (currentDataType.getCode() != DataType.Code.STRUCT) { return Optional.empty(); @@ -502,7 +499,7 @@ public Optional resolveIdentifierInType(@Nonnull Identifier requeste } } // probably need to check if currentDataType = targetDataType - final var fieldPath = FieldValue.resolveFieldPath(DataTypeUtils.toRecordLayerType(existingDataType), accessors.build()); + final var fieldPath = FieldValue.resolveFieldPath(objectValue.getResultType(), accessors.build()); final var attributeExpression = FieldValue.ofFieldsAndFuseIfPossible(objectValue, fieldPath); return Optional.of(attributeExpression); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 7b9591eacd..6ee3f0b1c0 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -22,13 +22,17 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalSortExpression; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; +import com.apple.foundationdb.record.query.plan.cascades.values.ObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.ddl.MetadataOperationsFactory; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.generated.RelationalParser; +import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerColumn; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; @@ -179,27 +183,21 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi @Nonnull @Override public UserDefinedFunctionDefinition visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { - // id = x.latitude final var id = visitFullId(ctx.fullId()); final var paramNameId = Identifier.of(ctx.paramName.getText().toUpperCase(Locale.ROOT)); - // (TODO) if function = return x - // newId = latitude, x is removed - // final var newId = id.removePrefix(paramNameId); final var ddlCatalog = metadataBuilder.build(); // parse the function definition using the newly constructed metadata. getDelegate().replaceCatalog(ddlCatalog); final var semanticAnalyzer = getDelegate().getSemanticAnalyzer(); - // columnType = DataType Location + final var inputcolumnTypeId = ctx.columnType(0).customType != null ? visitUid(ctx.columnType(0).customType) : Identifier.of(ctx.columnType(0).getText()); final var columnType = semanticAnalyzer.lookupType(inputcolumnTypeId, true, false, metadataBuilder::findType); - // look up the identifier in a type - // for example, inputParamType = Person(string name, Location address) Location(string street, string zipcode) - // resolveIdentifierInType("street", Person) -> Expression("name"="street", dataType=Person, underlying=FieldValue(PARAM.address.street) -> keyExpression = field("address").nest("street")) - Optional expression = semanticAnalyzer.resolveIdentifierInType(id, paramNameId, columnType); + ObjectValue argumentValue = ObjectValue.of(CorrelationIdentifier.of("PARAM"), DataTypeUtils.toRecordLayerType(columnType)); + Optional expression = semanticAnalyzer.resolveIdentifierInType(id, paramNameId, argumentValue); Assert.thatUnchecked(expression.isPresent(), "couldn't resolve function definition"); - return new UserDefinedFunctionDefinition(ctx.functionName.getText(), expression.get()); + return new UserDefinedFunctionDefinition(ctx.functionName.getText(), expression.get(), argumentValue); } @Nonnull diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index 82a335ce39..de24707736 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -876,7 +876,7 @@ void aliasingTableToResolveAmbiguityWorks() throws Exception { } } - // (TODO): more tests: 1) invalid param type lat(x loc) 2) return x + // (TODO): more tests: 1) invalid param type lat(x loc) 2) return (x+1) @Test void testUserDefinedFunction() throws Exception { final String schemaTemplate = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + @@ -923,7 +923,6 @@ void testUserDefinedFunction2() throws Exception { } } - // (TODO) not working yet, can fieldPath be empty? @Test void testUserDefinedFunction3() throws Exception { final String schemaTemplate = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + From eee22fcddf0988c0c38dda7c4f9ab5be68aa4231 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 12 Dec 2024 19:42:40 -0800 Subject: [PATCH 22/41] reformat --- .../recordlayer/query/SemanticAnalyzer.java | 16 +++---- .../query/visitors/BaseVisitor.java | 1 - .../query/visitors/DdlVisitor.java | 46 ++++--------------- .../query/visitors/ExpressionVisitor.java | 2 - .../recordlayer/query/StandardQueryTests.java | 13 +++--- 5 files changed, 24 insertions(+), 54 deletions(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index 8e2bf5b774..368d7d8af1 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -374,7 +374,7 @@ private List lookup(@Nonnull Identifier referenceIdentifier, continue; } } - final var nestedFieldMaybe = resolveIdentifierInType(referenceIdentifier, attribute, operator, matchQualifiedOnly); + final var nestedFieldMaybe = lookUpNestedField(referenceIdentifier, attribute, operator, matchQualifiedOnly); if (nestedFieldMaybe.isPresent()) { matchedAttributes.add(nestedFieldMaybe.get()); checkForPseudoColumns = false; @@ -414,10 +414,10 @@ public Optional lookupAlias(@Nonnull Identifier requestedAlias, } @Nonnull - public Optional resolveIdentifierInType(@Nonnull Identifier requestedIdentifier, - @Nonnull Expression existingExpression, - @Nonnull LogicalOperator logicalOperator, - boolean matchQualifiedOnly) { + public Optional lookUpNestedField(@Nonnull Identifier requestedIdentifier, + @Nonnull Expression existingExpression, + @Nonnull LogicalOperator logicalOperator, + boolean matchQualifiedOnly) { if (existingExpression.getName().isEmpty() || requestedIdentifier.fullyQualifiedName().size() <= 1) { return Optional.empty(); } @@ -469,9 +469,9 @@ public Optional resolveIdentifierInType(@Nonnull Identifier requeste } @Nonnull - public Optional resolveIdentifierInType(@Nonnull Identifier requestedIdentifier, - @Nonnull Identifier paramId, - @Nonnull ObjectValue objectValue) { + public Optional lookUpNestedField(@Nonnull Identifier requestedIdentifier, + @Nonnull Identifier paramId, + @Nonnull ObjectValue objectValue) { Assert.thatUnchecked(requestedIdentifier.prefixedWith(paramId), "Invalid function definition"); if (requestedIdentifier.fullyQualifiedName().size() == paramId.fullyQualifiedName().size()) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index e38274c260..cc1a4c472e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -180,7 +180,6 @@ boolean isTopLevel() { @Nonnull LogicalPlanFragment pushPlanFragment() { - System.out.println("pushPlanFragment is called"); currentPlanFragment = Optional.of(currentPlanFragment.map(LogicalPlanFragment::addChild).orElse(LogicalPlanFragment.ofRoot())); return currentPlanFragment.get(); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 6ee3f0b1c0..092f1e32b8 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -24,7 +24,6 @@ import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalSortExpression; -import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.ObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.relational.api.Options; @@ -51,7 +50,6 @@ import javax.annotation.Nonnull; import java.net.URI; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Locale; import java.util.Optional; @@ -183,21 +181,22 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi @Nonnull @Override public UserDefinedFunctionDefinition visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { - final var id = visitFullId(ctx.fullId()); - final var paramNameId = Identifier.of(ctx.paramName.getText().toUpperCase(Locale.ROOT)); - final var ddlCatalog = metadataBuilder.build(); // parse the function definition using the newly constructed metadata. getDelegate().replaceCatalog(ddlCatalog); final var semanticAnalyzer = getDelegate().getSemanticAnalyzer(); - final var inputcolumnTypeId = ctx.columnType(0).customType != null ? visitUid(ctx.columnType(0).customType) : Identifier.of(ctx.columnType(0).getText()); - final var columnType = semanticAnalyzer.lookupType(inputcolumnTypeId, true, false, metadataBuilder::findType); - + final var inputColumnId = ctx.columnType(0).customType != null ? visitUid(ctx.columnType(0).customType) : Identifier.of(ctx.columnType(0).getText()); + final var columnType = semanticAnalyzer.lookupType(inputColumnId, true, false, metadataBuilder::findType); ObjectValue argumentValue = ObjectValue.of(CorrelationIdentifier.of("PARAM"), DataTypeUtils.toRecordLayerType(columnType)); - Optional expression = semanticAnalyzer.resolveIdentifierInType(id, paramNameId, argumentValue); - Assert.thatUnchecked(expression.isPresent(), "couldn't resolve function definition"); - return new UserDefinedFunctionDefinition(ctx.functionName.getText(), expression.get(), argumentValue); + + final var id = visitFullId(ctx.fullId()); + final var paramNameId = Identifier.of(ctx.paramName.getText().toUpperCase(Locale.ROOT)); + + Optional functionValue = semanticAnalyzer.lookUpNestedField(id, paramNameId, argumentValue); + Assert.thatUnchecked(functionValue.isPresent(), "couldn't resolve function definition"); + + return new UserDefinedFunctionDefinition(ctx.functionName.getText(), functionValue.get(), argumentValue); } @Nonnull @@ -321,29 +320,4 @@ public ProceduralPlan visitDropSchemaTemplateStatement(@Nonnull RelationalParser public Boolean visitNullColumnConstraint(@Nonnull RelationalParser.NullColumnConstraintContext ctx) { return ctx.nullNotnull().NOT() == null; } - - private static RecordMetaDataProto.KeyExpression functionDefinitionToKeyExpression(String paramName, RelationalParser.ExpressionContext expressionContext) { - String[] fieldPath = expressionContext.getText().split("\\."); - Assert.thatUnchecked(fieldPath[0].equals(paramName), "Invalid function definition"); - fieldPath[0] = "PARAM"; - return arrayToKeyExpression(fieldPath); - } - - @Nonnull - private static RecordMetaDataProto.KeyExpression arrayToKeyExpression(@Nonnull String[] nameArray) { - RecordMetaDataProto.KeyExpression.Builder builder = RecordMetaDataProto.KeyExpression.newBuilder(); - if (nameArray.length == 1) { - return builder.setField(RecordMetaDataProto.Field.newBuilder() - .setFieldName(nameArray[0]) - .setFanType(RecordMetaDataProto.Field.FanType.SCALAR)) - .build(); - } else { - return builder.setNesting(RecordMetaDataProto.Nesting.newBuilder() - .setParent(RecordMetaDataProto.Field.newBuilder() - .setFieldName(nameArray[0]) - .setFanType(RecordMetaDataProto.Field.FanType.SCALAR)) - .setChild(arrayToKeyExpression(Arrays.copyOfRange(nameArray, 1, nameArray.length)))) - .build(); - } - } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index e4ea7bb3b9..b837de6e38 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -258,8 +258,6 @@ public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDef // special case for user-defined functions where we want to exclude the first argument from // being literal-stripped. @Nonnull Expressions arguments; - // check if this function exists in schema template - UDF udf = getDelegate().getCatalog().getUDF(functionName); // look up in FunctionCatalog boolean isUdf = getDelegate().getSemanticAnalyzer().isUdfFunction(functionName); if (isUdf) { diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index de24707736..ba8e3ec70a 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -876,19 +876,18 @@ void aliasingTableToResolveAmbiguityWorks() throws Exception { } } - // (TODO): more tests: 1) invalid param type lat(x loc) 2) return (x+1) + // (TODO): more tests: 1) invalid param type lat(x loc) 2) return (x+1) 3) nested functions @Test void testUserDefinedFunction() throws Exception { final String schemaTemplate = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n" + - "CREATE FUNCTION name(x Location) RETURNS string AS x.name\n" + - "CREATE FUNCTION latlon(x LATLON) RETURNS LATLON AS x"; + "CREATE FUNCTION name(x Location) RETURNS string AS x.name\n"; try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { try (var s = ddl.setSchemaAndGetConnection().createStatement()) { - insertLocationComplexRecord2(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); + insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); } // (TODO): double quote around loc, to differentiate udf from scalar function call, better way of doing it? try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE name((loc)) = ?name")) { @@ -912,7 +911,7 @@ void testUserDefinedFunction2() throws Exception { try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { try (var s = ddl.setSchemaAndGetConnection().createStatement()) { - insertLocationComplexRecord2(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); + insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); } try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE lat(loc) = ?loc")) { ps.setString("loc", "37.3"); @@ -934,7 +933,7 @@ void testUserDefinedFunction3() throws Exception { try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { try (var s = ddl.setSchemaAndGetConnection().createStatement()) { - insertLocationComplexRecord2(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); + insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); } // (TODO): double quote around loc, to differentiate udf from scalar function call, better way of doing it? try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE id((uid)) = ?id")) { @@ -949,7 +948,7 @@ void testUserDefinedFunction3() throws Exception { } } - private RelationalStruct insertLocationComplexRecord2(RelationalStatement s, long uid, @Nonnull final String name, String latitude, String longitude) throws SQLException { + private RelationalStruct insertLocationComplexRecord(RelationalStatement s, long uid, @Nonnull final String name, String latitude, String longitude) throws SQLException { var coord = EmbeddedRelationalStruct.newBuilder() .addString("LATITUDE", latitude) .addString("LONGITUDE", longitude) From 41112ed68356c5c072932a716a5718555e5ecc88 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 19 Dec 2024 16:25:07 -0800 Subject: [PATCH 23/41] fix parser --- .../src/main/antlr/RelationalLexer.g4 | 1 + .../src/main/antlr/RelationalParser.g4 | 11 ++- .../recordlayer/CatalogMetaData.java | 9 ++- .../metadata/RecordLayerSchemaTemplate.java | 29 ++++---- .../UserDefinedFunctionDefinition.java | 37 ++++------ .../serde/RecordMetadataDeserializer.java | 3 + .../serde/RecordMetadataSerializer.java | 1 + .../recordlayer/query/AstNormalizer.java | 1 + .../recordlayer/query/SemanticAnalyzer.java | 24 ++---- .../query/visitors/BaseVisitor.java | 6 ++ .../query/visitors/DdlVisitor.java | 20 ++--- .../query/visitors/DelegatingVisitor.java | 6 ++ .../query/visitors/ExpressionVisitor.java | 29 +------- .../query/visitors/TypedVisitor.java | 4 + .../relational/util/NullableArrayUtils.java | 55 +++++++------- .../src/main/proto/continuation.proto | 1 - .../api/ddl/DdlStatementParsingTest.java | 13 ++-- .../query/NullableArrayUtilsTest.java | 73 ++++++++++--------- .../recordlayer/query/StandardQueryTests.java | 10 +-- 19 files changed, 157 insertions(+), 176 deletions(-) diff --git a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 index 29802d8b3c..993bbeb4ff 100644 --- a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 @@ -712,6 +712,7 @@ TRANSACTIONAL: 'TRANSACTIONAL'; TRIGGERS: 'TRIGGERS'; TRUNCATE: 'TRUNCATE'; TYPE: 'TYPE'; +UDF_: 'UDF_'; UNBOUNDED: 'UNBOUNDED'; UNDEFINED: 'UNDEFINED'; UNDOFILE: 'UNDOFILE'; diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index b13bf415ad..62649d60b2 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -564,6 +564,11 @@ uid | DOUBLE_QUOTE_ID ; +userDefinedFunctionName + : ID + | DOUBLE_QUOTE_ID + ; + // done simpleId : ID @@ -792,8 +797,8 @@ ifNotExists functionCall : aggregateWindowedFunction #aggregateFunctionCall // done (supported) | specificFunction #specificFunctionCall // - | scalarFunctionName '(' functionArgs? ')' #scalarFunctionCall // done (unsupported) - | userDefinedFunctionName=uid '(' functionArgs? ')' #userDefinedFunctionCall + | scalarFunctionName '(' functionArgs? ')' #scalarFunctionCall // done (unsupported) + | userDefinedFunctionName '(' functionArgs? ')' #userDefinedFunctionCall ; specificFunction @@ -904,7 +909,7 @@ levelInWeightListElement ; aggregateWindowedFunction - : functionName=(AVG | MAX | MIN | SUM | MAX_EVER | MIN_EVER ) + : functionName=(AVG | MAX | MIN | SUM | MAX_EVER | MIN_EVER) '(' aggregator=(ALL | DISTINCT)? functionArg ')' overClause? | functionName=BITMAP_CONSTRUCT_AGG '(' functionArg ')' | functionName=COUNT '(' (starArg='*' | aggregator=ALL? functionArg | aggregator=DISTINCT functionArgs) ')' overClause? diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java index ead8f3b62a..ae880f0cd6 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.RecordMetaData; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.MetaDataException; @@ -155,7 +156,7 @@ public RelationalResultSet getPrimaryKeys(String database, String schema, String Stream rows = schemaInfo.getRecordTypesList().stream() .filter(type -> type.getName().equals(table)) .map(type -> { - RecordMetaDataProto.KeyExpression ke = type.getPrimaryKey(); + RecordKeyExpressionProto.KeyExpression ke = type.getPrimaryKey(); return new AbstractMap.SimpleEntry<>(type.getName(), keyExpressionToPrimaryKey(ke)); }).flatMap(pks -> IntStream.range(0, pks.getValue().length) .mapToObj(pos -> new ArrayRow(database, @@ -347,12 +348,12 @@ private RecordMetaDataProto.MetaData loadSchemaMetadata(@Nonnull final String da } //the position in the array is the key sequence, the value is the name of the column - private String[] keyExpressionToPrimaryKey(RecordMetaDataProto.KeyExpression ke) throws UncheckedRelationalException { + private String[] keyExpressionToPrimaryKey(RecordKeyExpressionProto.KeyExpression ke) throws UncheckedRelationalException { if (ke.hasThen()) { - final List childList = ke.getThen().getChildList(); + final List childList = ke.getThen().getChildList(); String[] fields = new String[childList.size()]; int pos = 0; - for (RecordMetaDataProto.KeyExpression childKe : childList) { + for (RecordKeyExpressionProto.KeyExpression childKe : childList) { //skip record type keys if (!childKe.hasRecordTypeKey()) { if (childKe.hasField()) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java index e8f3ea5dfd..27db340fbd 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java @@ -25,6 +25,7 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.Key; +import com.apple.foundationdb.record.metadata.Udf; import com.apple.foundationdb.record.query.combinatorics.TopologicalSort; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.exceptions.RelationalException; @@ -71,6 +72,8 @@ public final class RecordLayerSchemaTemplate implements SchemaTemplate { @Nonnull private final Set tables; + @Nonnull + private final Set udfs; private final int version; private final boolean enableLongRows; @@ -88,6 +91,7 @@ public final class RecordLayerSchemaTemplate implements SchemaTemplate { private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, + @Nonnull final Set udfs, int version, boolean enableLongRows, boolean storeRowVersions) { @@ -103,6 +107,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, + @Nonnull final Set udfs, int version, boolean enableLongRows, boolean storeRowVersions, @@ -150,17 +155,9 @@ public RecordLayerSchema generateSchema(@Nonnull String databaseId, @Nonnull Str return new RecordLayerSchema(schemaName, databaseId, this); } - @Nullable - public FunctionDefinition getOperationDefinition(@Nonnull final String operatorName, @Nonnull final DataType.StructType structType) { - for (RecordLayerTable table: tables) { - for (RecordLayerColumn column: table.getColumns()) { - final var dt = column.getDataType(); - if (dt instanceof DataType.StructType && dt.equals(structType) && ((DataType.StructType) dt).getUserDefinedFunctions().containsKey(operatorName)) { - return ((DataType.StructType) dt).getUserDefinedFunctions().get(operatorName); - } - } - } - return null; + @Nonnull + public Set getAllUdfs() { + return udfs; } @Nonnull @@ -421,15 +418,15 @@ public Builder addAuxiliaryTypes(@Nonnull Collection auxiliaryTy } @Nonnull - public Builder addUDF(@Nonnull UDF udf) { - Assert.thatUnchecked(!udfMap.containsKey(udf.getUdfName()), ErrorCode.INVALID_SCHEMA_TEMPLATE, TABLE_ALREADY_EXISTS, udf.getUdfName()); - udfMap.put(udf.getUdfName(), udf); + public Builder addUdf(@Nonnull Udf Udf) { + Assert.thatUnchecked(!udfMap.containsKey(Udf.getFunctionName()), ErrorCode.INVALID_SCHEMA_TEMPLATE, TABLE_ALREADY_EXISTS, Udf.getFunctionName()); + udfMap.put(Udf.getFunctionName(), Udf); return this; } @Nonnull - public Builder addUDFs(@Nonnull Collection udf) { - udf.forEach(this::addUDF); + public Builder addUdfs(@Nonnull Collection Udf) { + Udf.forEach(this::addUdf); return this; } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java index 286d3e4a07..998b26694e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java @@ -20,24 +20,23 @@ package com.apple.foundationdb.relational.recordlayer.metadata; -import com.apple.foundationdb.record.metadata.UDF; +import com.apple.foundationdb.record.metadata.Udf; import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; -import com.apple.foundationdb.record.query.plan.cascades.values.ObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.MacroFunctionValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import javax.annotation.Nonnull; +import java.util.ArrayList; import java.util.List; public class UserDefinedFunctionDefinition { @Nonnull private final String functionName; - @Nonnull private final Value value; - @Nonnull private final ObjectValue argumentValue; + @Nonnull private final Value functionValue; - public UserDefinedFunctionDefinition(@Nonnull String functionName, @Nonnull Value value, @Nonnull ObjectValue argumentValue) { + public UserDefinedFunctionDefinition(@Nonnull String functionName, @Nonnull Value value) { this.functionName = functionName; - this.value = value; - this.argumentValue = argumentValue; + this.functionValue = value; } @Nonnull @@ -46,25 +45,17 @@ public String getName() { } public BuiltInFunction getBuiltInFunction() { - return new BuiltInFunction(functionName, List.of(), (builtInFunction, arguments) -> replaceArgument((Value) arguments.get(0))) {}; + return new BuiltInFunction(functionName, List.of(), (builtInFunction, arguments) -> { + final var result = ((MacroFunctionValue) functionValue).call(arguments); + return result; + }) {}; } - public UDF toUDF() { - return new UDF(functionName, value, argumentValue); + public Udf toUDF() { + return new Udf(functionName, functionValue); } - public static UserDefinedFunctionDefinition fromUDF(UDF udf) { - return new UserDefinedFunctionDefinition(udf.getUdfName(), udf.getValue(), (ObjectValue) udf.getArgumentValue()); - } - - // replace the argumentValue with argument in value - private Value replaceArgument(Value argument) { - return value.replace((v) -> { - if (v instanceof ObjectValue && v.getResultType().equals(argumentValue.getResultType())) { - return argument; - } else { - return v; - } - }); + public static UserDefinedFunctionDefinition fromUdf(Udf udf) { + return new UserDefinedFunctionDefinition(udf.getFunctionName(), udf.getValue()); } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java index 1a18fb082d..b7e8403957 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java @@ -60,6 +60,9 @@ public RecordLayerSchemaTemplate.Builder getSchemaTemplate(@Nonnull final String .setEnableLongRows(recordMetaData.isSplitLongRecords()) .setName(schemaTemplateName) .setIntermingleTables(!recordMetaData.primaryKeyHasRecordTypePrefix()); + for (final var u: recordMetaData.getAllUdfs()) { + schemaTemplateBuilder.addUdf(u); + } final var nameToTableBuilder = new HashMap(); for (final var registeredType : registeredTypes) { switch (registeredType.getType()) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java index a668315e41..830a1ce342 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java @@ -87,6 +87,7 @@ public void visit(@Nonnull com.apple.foundationdb.relational.api.metadata.Index @Override public void visit(@Nonnull SchemaTemplate schemaTemplate) { Assert.thatUnchecked(schemaTemplate instanceof RecordLayerSchemaTemplate); + getBuilder().addUdfs(((RecordLayerSchemaTemplate) schemaTemplate).getAllUdfs()); getBuilder().setSplitLongRecords(schemaTemplate.isEnableLongRows()); getBuilder().setStoreRecordVersions(schemaTemplate.isStoreRowVersions()); getBuilder().setVersion(schemaTemplate.getVersion()); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java index f89278b3fb..98bbdb9a55 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java @@ -351,6 +351,7 @@ public Void visitContinuationAtom(@Nonnull RelationalParser.ContinuationAtomCont @Override @SuppressWarnings("PMD.CompareObjectsWithEquals") // deliberate use of pointer equality public Void visitScalarFunctionCall(@Nonnull RelationalParser.ScalarFunctionCallContext ctx) { + // final var functionName = ctx.scalarFunctionName() == null ? ctx.userDefinedFunctionName.getText() : ctx.scalarFunctionName().getText(); final var functionName = ctx.scalarFunctionName().getText(); boolean skipFirstFunctionArgument = "JAVA_CALL".equals(SemanticAnalyzer.normalizeString(functionName, false)); for (int i = 0; i < ctx.getChildCount(); i++) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index 368d7d8af1..f5513529cb 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -21,7 +21,7 @@ package com.apple.foundationdb.relational.recordlayer.query; import com.apple.foundationdb.annotation.API; - +import com.apple.foundationdb.record.metadata.Udf; import com.apple.foundationdb.record.query.plan.cascades.AccessHint; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; import com.apple.foundationdb.record.query.plan.cascades.Correlated; @@ -30,21 +30,7 @@ import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; -import com.apple.foundationdb.record.query.plan.cascades.values.AggregateValue; -import com.apple.foundationdb.record.query.plan.cascades.values.AndOrValue; -import com.apple.foundationdb.record.query.plan.cascades.values.ArithmeticValue; -import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; -import com.apple.foundationdb.record.query.plan.cascades.values.InOpValue; -import com.apple.foundationdb.record.query.plan.cascades.values.IndexableAggregateValue; -import com.apple.foundationdb.record.query.plan.cascades.values.JavaCallFunction; -import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; -import com.apple.foundationdb.record.query.plan.cascades.values.NotValue; -import com.apple.foundationdb.record.query.plan.cascades.values.ObjectValue; -import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; -import com.apple.foundationdb.record.query.plan.cascades.values.RelOpValue; -import com.apple.foundationdb.record.query.plan.cascades.values.StreamableAggregateValue; -import com.apple.foundationdb.record.query.plan.cascades.values.Value; -import com.apple.foundationdb.record.util.pair.NonnullPair; +import com.apple.foundationdb.record.query.plan.cascades.values.*; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.api.metadata.DataType; @@ -96,6 +82,10 @@ public SemanticAnalyzer(@Nonnull SchemaTemplate metadataCatalog, @Nonnull FunctionCatalog functionCatalog) { this.metadataCatalog = metadataCatalog; this.functionCatalog = functionCatalog; + // add UDFs to functionCatalog + for (Udf udf: ((RecordLayerSchemaTemplate) metadataCatalog).getAllUdfs()) { + ((SqlFunctionCatalog)this.functionCatalog).addFunction(UserDefinedFunctionDefinition.fromUdf(udf)); + } } /** @@ -471,7 +461,7 @@ public Optional lookUpNestedField(@Nonnull Identifier requestedIdent @Nonnull public Optional lookUpNestedField(@Nonnull Identifier requestedIdentifier, @Nonnull Identifier paramId, - @Nonnull ObjectValue objectValue) { + @Nonnull QuantifiedObjectValue objectValue) { Assert.thatUnchecked(requestedIdentifier.prefixedWith(paramId), "Invalid function definition"); if (requestedIdentifier.fullyQualifiedName().size() == paramId.fullyQualifiedName().size()) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index cc1a4c472e..36d7fde3fc 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -1337,6 +1337,12 @@ public Object visitScalarFunctionName(@Nonnull RelationalParser.ScalarFunctionNa return visitChildren(ctx); } + @Nonnull + @Override + public Object visitUserDefinedFunctionName(@Nonnull RelationalParser.UserDefinedFunctionNameContext ctx) { + return visitChildren(ctx); + } + @Nonnull @Override public Expressions visitFunctionArgs(@Nonnull RelationalParser.FunctionArgsContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 092f1e32b8..ce258ca231 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -24,7 +24,8 @@ import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalSortExpression; -import com.apple.foundationdb.record.query.plan.cascades.values.ObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.MacroFunctionValue; +import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.relational.api.Options; import com.apple.foundationdb.relational.api.ddl.MetadataOperationsFactory; @@ -188,15 +189,15 @@ public UserDefinedFunctionDefinition visitFunctionDefinition(@Nonnull Relational final var inputColumnId = ctx.columnType(0).customType != null ? visitUid(ctx.columnType(0).customType) : Identifier.of(ctx.columnType(0).getText()); final var columnType = semanticAnalyzer.lookupType(inputColumnId, true, false, metadataBuilder::findType); - ObjectValue argumentValue = ObjectValue.of(CorrelationIdentifier.of("PARAM"), DataTypeUtils.toRecordLayerType(columnType)); + QuantifiedObjectValue argumentValue = QuantifiedObjectValue.of(CorrelationIdentifier.of(ctx.paramName.getText()), DataTypeUtils.toRecordLayerType(columnType)); final var id = visitFullId(ctx.fullId()); final var paramNameId = Identifier.of(ctx.paramName.getText().toUpperCase(Locale.ROOT)); - Optional functionValue = semanticAnalyzer.lookUpNestedField(id, paramNameId, argumentValue); - Assert.thatUnchecked(functionValue.isPresent(), "couldn't resolve function definition"); + Optional fieldValue = semanticAnalyzer.lookUpNestedField(id, paramNameId, argumentValue); + Assert.thatUnchecked(fieldValue.isPresent(), "couldn't resolve function definition"); - return new UserDefinedFunctionDefinition(ctx.functionName.getText(), functionValue.get(), argumentValue); + return new UserDefinedFunctionDefinition(ctx.functionName.getText(), MacroFunctionValue.of(List.of(argumentValue), fieldValue.get())); } @Nonnull @@ -259,13 +260,8 @@ public ProceduralPlan visitCreateSchemaTemplateStatement(@Nonnull RelationalPars metadataBuilder.addTable(tableWithIndex); } final var functions = functionClauses.build().stream().map(this::visitFunctionDefinition).collect(ImmutableList.toImmutableList()); - for (final var func : functions) { - // (TODO): maybe a table type too? - final var auxiliaryType = metadataBuilder.extractAuxiliaryType(func.getInputTypeName()); - System.out.println("auxiliaryType:" + auxiliaryType); - Assert.thatUnchecked(auxiliaryType instanceof DataType.StructType); - metadataBuilder.addAuxiliaryType(((DataType.StructType) auxiliaryType).withFunctionDefinition(func)); - } + final var udfs = functions.stream().map(v -> v.toUDF()).collect(Collectors.toList()); + metadataBuilder.addUdfs(udfs); return ProceduralPlan.of(metadataOperationsFactory.getCreateSchemaTemplateConstantAction(metadataBuilder.build(), Options.NONE)); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index 114832a184..f2b09f4dfb 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -1183,6 +1183,12 @@ public Object visitScalarFunctionName(@Nonnull RelationalParser.ScalarFunctionNa return getDelegate().visitScalarFunctionName(ctx); } + @Nonnull + @Override + public Object visitUserDefinedFunctionName(@Nonnull RelationalParser.UserDefinedFunctionNameContext ctx) { + return getDelegate().visitUserDefinedFunctionName(ctx); + } + @Nonnull @Override public Expressions visitFunctionArgs(@Nonnull RelationalParser.FunctionArgsContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index b837de6e38..cb4d6752b9 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -21,7 +21,7 @@ package com.apple.foundationdb.relational.recordlayer.query.visitors; import com.apple.foundationdb.annotation.API; - +import com.apple.foundationdb.record.metadata.Udf; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.AbstractArrayConstructorValue; @@ -219,7 +219,7 @@ public Expression visitScalarFunctionCall(@Nonnull RelationalParser.ScalarFuncti final var classNameExpression = getDelegate().getPlanGenerationContext().withDisabledLiteralProcessing(() -> { final var result = visitFunctionArg(argumentNodes.get(0)); Assert.thatUnchecked(result.getUnderlying() instanceof LiteralValue, - ErrorCode.INVALID_ARGUMENT_FOR_FUNCTION, () -> String.format("attempt to invoke java_call with incorrect UDF '%s'", + ErrorCode.INVALID_ARGUMENT_FOR_FUNCTION, () -> String.format("attempt to invoke java_call with incorrect Udf '%s'", result.getUnderlying())); return result; }); @@ -254,31 +254,10 @@ public Expression visitCaseFunctionCall(@Nonnull RelationalParser.CaseFunctionCa @Nonnull @Override public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { - final var functionName = ctx.userDefinedFunctionName.getText(); + final var functionName = ctx.userDefinedFunctionName().getText(); // special case for user-defined functions where we want to exclude the first argument from // being literal-stripped. - @Nonnull Expressions arguments; - // look up in FunctionCatalog - boolean isUdf = getDelegate().getSemanticAnalyzer().isUdfFunction(functionName); - if (isUdf) { - final var argumentNodes = ctx.functionArgs().children.stream() - .filter(arg -> arg instanceof RelationalParser.FunctionArgContext) - .map(RelationalParser.FunctionArgContext.class::cast) - .collect(Collectors.toUnmodifiableList()); - Assert.thatUnchecked(!argumentNodes.isEmpty()); - final var classNameExpression = getDelegate().getPlanGenerationContext().withDisabledLiteralProcessing(() -> { - final var result = visitFunctionArg(argumentNodes.get(0)); - Assert.thatUnchecked(result.getUnderlying() instanceof LiteralValue, - ErrorCode.INVALID_ARGUMENT_FOR_FUNCTION, () -> String.format("attempt to invoke java_call with incorrect UDF '%s'", - result.getUnderlying())); - return result; - }); - arguments = Expressions.of(Streams.concat(Stream.of(classNameExpression), - argumentNodes.stream().skip(1).map(this::visitFunctionArg)) - .collect(Collectors.toUnmodifiableList())); - } else { - arguments = visitFunctionArgs(ctx.functionArgs()); - } + @Nonnull Expressions arguments = visitFunctionArgs(ctx.functionArgs()); return getDelegate().resolveFunction(functionName, arguments.asList().toArray(new Expression[0])); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java index a12cbb0f18..546f271a90 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java @@ -790,6 +790,10 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override Object visitScalarFunctionName(@Nonnull RelationalParser.ScalarFunctionNameContext ctx); + @Nonnull + @Override + Object visitUserDefinedFunctionName(@Nonnull RelationalParser.UserDefinedFunctionNameContext ctx); + @Nonnull @Override Expressions visitFunctionArgs(@Nonnull RelationalParser.FunctionArgsContext ctx); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/util/NullableArrayUtils.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/util/NullableArrayUtils.java index 86b85b689f..213e487550 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/util/NullableArrayUtils.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/util/NullableArrayUtils.java @@ -21,8 +21,7 @@ package com.apple.foundationdb.relational.util; import com.apple.foundationdb.annotation.API; - -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; @@ -70,9 +69,9 @@ public static boolean isWrappedArrayDescriptor(@Nonnull final Descriptors.Descri * * TODO Add the wrapped array structure for nullable arrays. */ - public static RecordMetaDataProto.KeyExpression wrapArray(RecordMetaDataProto.KeyExpression keyExpression, - final Type.Record record, - boolean containsNullableArray) { + public static RecordKeyExpressionProto.KeyExpression wrapArray(RecordKeyExpressionProto.KeyExpression keyExpression, + final Type.Record record, + boolean containsNullableArray) { if (!containsNullableArray) { return keyExpression; } @@ -87,7 +86,7 @@ public static RecordMetaDataProto.KeyExpression wrapArray(RecordMetaDataProto.Ke For example, reviews.rating -> reviews.values.rating (TODO): Add the wrapped array structure for nullable arrays. */ - public static RecordMetaDataProto.KeyExpression wrapArray(RecordMetaDataProto.KeyExpression keyExpression, + public static RecordKeyExpressionProto.KeyExpression wrapArray(RecordKeyExpressionProto.KeyExpression keyExpression, Descriptors.Descriptor parentDescriptor, boolean containsNullableArray) { if (!containsNullableArray) { @@ -97,15 +96,15 @@ public static RecordMetaDataProto.KeyExpression wrapArray(RecordMetaDataProto.Ke return wrapArrayInternal(keyExpression, parentDescriptor); } - private static RecordMetaDataProto.KeyExpression wrapArrayInternal(RecordMetaDataProto.KeyExpression keyExpression, + private static RecordKeyExpressionProto.KeyExpression wrapArrayInternal(RecordKeyExpressionProto.KeyExpression keyExpression, Descriptors.Descriptor parentDescriptor) { // handle concat (straightforward recursion) if (keyExpression.hasThen()) { - final var newThenBuilder = RecordMetaDataProto.Then.newBuilder(); + final var newThenBuilder = RecordKeyExpressionProto.Then.newBuilder(); for (final var child : keyExpression.getThen().getChildList()) { newThenBuilder.addChild(wrapArrayInternal(child, parentDescriptor)); } - return RecordMetaDataProto.KeyExpression.newBuilder().setThen(newThenBuilder).build(); + return RecordKeyExpressionProto.KeyExpression.newBuilder().setThen(newThenBuilder).build(); } // handle nested field @@ -120,20 +119,20 @@ private static RecordMetaDataProto.KeyExpression wrapArrayInternal(RecordMetaDat final var wrappedChild = wrapArrayInternal(child, parentDescriptor.findFieldByName(parentFieldName).getMessageType().findFieldByName(REPEATED_FIELD_NAME).getMessageType()); // the child is actually a grand child (since parent->child is actually parent->values->child), fix that. - final var newChild = RecordMetaDataProto.KeyExpression.newBuilder() - .setNesting(RecordMetaDataProto.Nesting.newBuilder() + final var newChild = RecordKeyExpressionProto.KeyExpression.newBuilder() + .setNesting(RecordKeyExpressionProto.Nesting.newBuilder() .setParent(wrappedParent.getChild().getField()) .setChild(wrappedChild)) .build(); - return RecordMetaDataProto.KeyExpression.newBuilder() - .setNesting(RecordMetaDataProto.Nesting.newBuilder() + return RecordKeyExpressionProto.KeyExpression.newBuilder() + .setNesting(RecordKeyExpressionProto.Nesting.newBuilder() .setParent(wrappedParent.getParent()) .setChild(newChild)) .build(); } else { final var wrappedChild = wrapArrayInternal(child, parentDescriptor.findFieldByName(parentFieldName).getMessageType()); - return RecordMetaDataProto.KeyExpression.newBuilder() - .setNesting(RecordMetaDataProto.Nesting.newBuilder() + return RecordKeyExpressionProto.KeyExpression.newBuilder() + .setNesting(RecordKeyExpressionProto.Nesting.newBuilder() .setParent(parent) .setChild(wrappedChild)) .build(); @@ -143,7 +142,7 @@ private static RecordMetaDataProto.KeyExpression wrapArrayInternal(RecordMetaDat // check key expression's field if (keyExpression.hasField()) { if (NullableArrayUtils.isWrappedArrayDescriptor(keyExpression.getField().getFieldName(), parentDescriptor)) { - return RecordMetaDataProto.KeyExpression.newBuilder().setNesting(splitFieldIntoNestedWithValues(keyExpression.getField())).build(); + return RecordKeyExpressionProto.KeyExpression.newBuilder().setNesting(splitFieldIntoNestedWithValues(keyExpression.getField())).build(); } else { return keyExpression; } @@ -152,50 +151,50 @@ private static RecordMetaDataProto.KeyExpression wrapArrayInternal(RecordMetaDat // grouping key expression if (keyExpression.hasGrouping()) { final var newWholeKey = wrapArrayInternal(keyExpression.getGrouping().getWholeKey(), parentDescriptor); - return RecordMetaDataProto.KeyExpression.newBuilder().setGrouping(keyExpression.getGrouping().toBuilder().setWholeKey(newWholeKey)).build(); + return RecordKeyExpressionProto.KeyExpression.newBuilder().setGrouping(keyExpression.getGrouping().toBuilder().setWholeKey(newWholeKey)).build(); } // split key expression if (keyExpression.hasSplit()) { final var newJoined = wrapArrayInternal(keyExpression.getSplit().getJoined(), parentDescriptor); - return RecordMetaDataProto.KeyExpression.newBuilder().setSplit(keyExpression.getSplit().toBuilder().setJoined(newJoined)).build(); + return RecordKeyExpressionProto.KeyExpression.newBuilder().setSplit(keyExpression.getSplit().toBuilder().setJoined(newJoined)).build(); } // function key expression. if (keyExpression.hasFunction()) { final var newArguments = wrapArrayInternal(keyExpression.getFunction().getArguments(), parentDescriptor); - return RecordMetaDataProto.KeyExpression.newBuilder().setFunction(keyExpression.getFunction().toBuilder().setArguments(newArguments)).build(); + return RecordKeyExpressionProto.KeyExpression.newBuilder().setFunction(keyExpression.getFunction().toBuilder().setArguments(newArguments)).build(); } // covering key expression. if (keyExpression.hasKeyWithValue()) { final var newInnerKey = wrapArrayInternal(keyExpression.getKeyWithValue().getInnerKey(), parentDescriptor); - return RecordMetaDataProto.KeyExpression.newBuilder().setKeyWithValue(keyExpression.getKeyWithValue().toBuilder().setInnerKey(newInnerKey)).build(); + return RecordKeyExpressionProto.KeyExpression.newBuilder().setKeyWithValue(keyExpression.getKeyWithValue().toBuilder().setInnerKey(newInnerKey)).build(); } // key expression containing list. if (keyExpression.hasList()) { - final var newListBuilder = RecordMetaDataProto.List.newBuilder(); + final var newListBuilder = RecordKeyExpressionProto.List.newBuilder(); for (final var listItem : keyExpression.getList().getChildList()) { newListBuilder.addChild(wrapArrayInternal(listItem, parentDescriptor)); } - return RecordMetaDataProto.KeyExpression.newBuilder().setList(newListBuilder).build(); + return RecordKeyExpressionProto.KeyExpression.newBuilder().setList(newListBuilder).build(); } return keyExpression; } // wrap repeated fields in a Field type keyExpression - private static RecordMetaDataProto.Nesting splitFieldIntoNestedWithValues(@Nonnull final RecordMetaDataProto.Field original) { - final var nestedArrayBuilder = RecordMetaDataProto.Field.newBuilder() + private static RecordKeyExpressionProto.Nesting splitFieldIntoNestedWithValues(@Nonnull final RecordKeyExpressionProto.Field original) { + final var nestedArrayBuilder = RecordKeyExpressionProto.Field.newBuilder() .setFieldName(original.getFieldName()) - .setFanType(RecordMetaDataProto.Field.FanType.SCALAR) + .setFanType(RecordKeyExpressionProto.Field.FanType.SCALAR) .setNullInterpretation(original.getNullInterpretation()); - final var arrayValueBuilder = RecordMetaDataProto.KeyExpression.newBuilder() - .setField(RecordMetaDataProto.Field.newBuilder() + final var arrayValueBuilder = RecordKeyExpressionProto.KeyExpression.newBuilder() + .setField(RecordKeyExpressionProto.Field.newBuilder() .setFieldName(REPEATED_FIELD_NAME) .setFanType(original.getFanType()) .setNullInterpretation(original.getNullInterpretation())); - return RecordMetaDataProto.Nesting.newBuilder().setParent(nestedArrayBuilder).setChild(arrayValueBuilder).build(); + return RecordKeyExpressionProto.Nesting.newBuilder().setParent(nestedArrayBuilder).setChild(arrayValueBuilder).build(); } } diff --git a/fdb-relational-core/src/main/proto/continuation.proto b/fdb-relational-core/src/main/proto/continuation.proto index 595140cae6..4b5dfae0d3 100644 --- a/fdb-relational-core/src/main/proto/continuation.proto +++ b/fdb-relational-core/src/main/proto/continuation.proto @@ -21,7 +21,6 @@ syntax = "proto3"; package com.apple.foundationdb.relational.continuation; import "record_metadata.proto"; -import "record_value.proto"; import "record_query_plan.proto"; import "record_query_runtime.proto"; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java index 562efe257e..1084071039 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.relational.api.ddl; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.RecordStoreState; @@ -408,7 +409,7 @@ public ConstantAction getCreateSchemaTemplateConstantAction(@Nonnull SchemaTempl Assertions.assertEquals("V_IDX", index.getName(), "Incorrect index name!"); final var actualKe = ((RecordLayerIndex) index).getKeyExpression().toKeyExpression(); - List keys = null; + List keys = null; if (actualKe.hasThen()) { keys = new ArrayList<>(actualKe.getThen().getChildList()); } else if (actualKe.hasField()) { @@ -453,14 +454,14 @@ public ConstantAction getCreateSchemaTemplateConstantAction(@Nonnull SchemaTempl final Index index = info.getIndexes().stream().findFirst().get(); Assertions.assertEquals("V_IDX", index.getName(), "Incorrect index name!"); - RecordMetaDataProto.KeyExpression actualKe = ((RecordLayerIndex) index).getKeyExpression().toKeyExpression(); + RecordKeyExpressionProto.KeyExpression actualKe = ((RecordLayerIndex) index).getKeyExpression().toKeyExpression(); Assertions.assertNotNull(actualKe.getKeyWithValue(), "Null KeyExpression for included columns!"); - final RecordMetaDataProto.KeyWithValue keyWithValue = actualKe.getKeyWithValue(); + final RecordKeyExpressionProto.KeyWithValue keyWithValue = actualKe.getKeyWithValue(); //This is a weird workaround for the problem fixed in https://github.com/FoundationDB/fdb-record-layer/pull/1585, // once that's been merged and we get a release that contains it, we can replace this with a more //natural api - final RecordMetaDataProto.KeyExpression innerKey = keyWithValue.getInnerKey(); + final RecordKeyExpressionProto.KeyExpression innerKey = keyWithValue.getInnerKey(); int splitPoint = keyWithValue.getSplitPoint(); final ThenKeyExpression then = new ThenKeyExpression(innerKey.getThen()); KeyExpression keyExpr = then.getSubKey(0, splitPoint); @@ -468,13 +469,13 @@ public ConstantAction getCreateSchemaTemplateConstantAction(@Nonnull SchemaTempl Assertions.assertEquals(indexedColumns.size(), keyExpr.getColumnSize(), "Incorrect number of parsed columns!"); for (int i = 0; i < indexedColumns.size(); i++) { - final RecordMetaDataProto.KeyExpression ke = keyExpr.getSubKey(i, i + 1).toKeyExpression(); + final RecordKeyExpressionProto.KeyExpression ke = keyExpr.getSubKey(i, i + 1).toKeyExpression(); Assertions.assertEquals(indexedColumns.get(i), ke.getField().getFieldName(), "Incorrect column at position " + i); } Assertions.assertEquals(unindexedColumns.size(), valueExpr.getColumnSize(), "Incorrect number of parsed columns!"); for (int i = 0; i < unindexedColumns.size(); i++) { - final RecordMetaDataProto.KeyExpression ve = valueExpr.getSubKey(i, i + 1).toKeyExpression(); + final RecordKeyExpressionProto.KeyExpression ve = valueExpr.getSubKey(i, i + 1).toKeyExpression(); Assertions.assertEquals(unindexedColumns.get(i), ve.getField().getFieldName(), "Incorrect column at position " + i); } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/NullableArrayUtilsTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/NullableArrayUtilsTest.java index 07db2a9a9e..3de4c7150a 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/NullableArrayUtilsTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/NullableArrayUtilsTest.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.relational.recordlayer.query; +import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.relational.util.NullableArrayUtils; @@ -55,56 +56,56 @@ void testWrapArray() throws Descriptors.DescriptorValidationException { Descriptors.FileDescriptor file = generateFileDescriptor(); Descriptors.Descriptor parentDescriptor = file.findMessageTypeByName("Parent"); // original field("stringListField", FAN_OUT), expected to become stringListField.values - RecordMetaDataProto.KeyExpression original1 = RecordMetaDataProto.KeyExpression.newBuilder() - .setField(RecordMetaDataProto.Field.newBuilder() + RecordKeyExpressionProto.KeyExpression original1 = RecordKeyExpressionProto.KeyExpression.newBuilder() + .setField(RecordKeyExpressionProto.Field.newBuilder() .setFieldName("stringListField") - .setFanType(RecordMetaDataProto.Field.FanType.FAN_OUT) - .setNullInterpretation(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE)) + .setFanType(RecordKeyExpressionProto.Field.FanType.FAN_OUT) + .setNullInterpretation(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE)) .build(); - RecordMetaDataProto.KeyExpression expected1 = RecordMetaDataProto.KeyExpression.newBuilder() - .setNesting(RecordMetaDataProto.Nesting.newBuilder() - .setParent(RecordMetaDataProto.Field.newBuilder() + RecordKeyExpressionProto.KeyExpression expected1 = RecordKeyExpressionProto.KeyExpression.newBuilder() + .setNesting(RecordKeyExpressionProto.Nesting.newBuilder() + .setParent(RecordKeyExpressionProto.Field.newBuilder() .setFieldName("stringListField") - .setFanType(RecordMetaDataProto.Field.FanType.SCALAR) - .setNullInterpretation(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE)) - .setChild(RecordMetaDataProto.KeyExpression.newBuilder() - .setField(RecordMetaDataProto.Field.newBuilder() + .setFanType(RecordKeyExpressionProto.Field.FanType.SCALAR) + .setNullInterpretation(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE)) + .setChild(RecordKeyExpressionProto.KeyExpression.newBuilder() + .setField(RecordKeyExpressionProto.Field.newBuilder() .setFieldName(NullableArrayUtils.getRepeatedFieldName()) - .setFanType(RecordMetaDataProto.Field.FanType.FAN_OUT) - .setNullInterpretation(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE)))) + .setFanType(RecordKeyExpressionProto.Field.FanType.FAN_OUT) + .setNullInterpretation(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE)))) .build(); Assertions.assertEquals(expected1, NullableArrayUtils.wrapArray(original1, parentDescriptor, true)); // userListField.name, expected to become userListField.values.name - RecordMetaDataProto.KeyExpression original2 = RecordMetaDataProto.KeyExpression.newBuilder() - .setNesting(RecordMetaDataProto.Nesting.newBuilder() - .setParent(RecordMetaDataProto.Field.newBuilder() + RecordKeyExpressionProto.KeyExpression original2 = RecordKeyExpressionProto.KeyExpression.newBuilder() + .setNesting(RecordKeyExpressionProto.Nesting.newBuilder() + .setParent(RecordKeyExpressionProto.Field.newBuilder() .setFieldName("userListField") - .setFanType(RecordMetaDataProto.Field.FanType.FAN_OUT) - .setNullInterpretation(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE)) - .setChild(RecordMetaDataProto.KeyExpression.newBuilder() - .setField(RecordMetaDataProto.Field.newBuilder() + .setFanType(RecordKeyExpressionProto.Field.FanType.FAN_OUT) + .setNullInterpretation(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE)) + .setChild(RecordKeyExpressionProto.KeyExpression.newBuilder() + .setField(RecordKeyExpressionProto.Field.newBuilder() .setFieldName("name") - .setFanType(RecordMetaDataProto.Field.FanType.SCALAR) - .setNullInterpretation(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE)))) + .setFanType(RecordKeyExpressionProto.Field.FanType.SCALAR) + .setNullInterpretation(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE)))) .build(); - RecordMetaDataProto.KeyExpression expected2 = RecordMetaDataProto.KeyExpression.newBuilder() - .setNesting(RecordMetaDataProto.Nesting.newBuilder() - .setParent(RecordMetaDataProto.Field.newBuilder() + RecordKeyExpressionProto.KeyExpression expected2 = RecordKeyExpressionProto.KeyExpression.newBuilder() + .setNesting(RecordKeyExpressionProto.Nesting.newBuilder() + .setParent(RecordKeyExpressionProto.Field.newBuilder() .setFieldName("userListField") - .setFanType(RecordMetaDataProto.Field.FanType.SCALAR) - .setNullInterpretation(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE)) - .setChild(RecordMetaDataProto.KeyExpression.newBuilder() - .setNesting(RecordMetaDataProto.Nesting.newBuilder() - .setParent(RecordMetaDataProto.Field.newBuilder() + .setFanType(RecordKeyExpressionProto.Field.FanType.SCALAR) + .setNullInterpretation(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE)) + .setChild(RecordKeyExpressionProto.KeyExpression.newBuilder() + .setNesting(RecordKeyExpressionProto.Nesting.newBuilder() + .setParent(RecordKeyExpressionProto.Field.newBuilder() .setFieldName(NullableArrayUtils.getRepeatedFieldName()) - .setFanType(RecordMetaDataProto.Field.FanType.FAN_OUT) - .setNullInterpretation(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE)) - .setChild(RecordMetaDataProto.KeyExpression.newBuilder() - .setField(RecordMetaDataProto.Field.newBuilder() + .setFanType(RecordKeyExpressionProto.Field.FanType.FAN_OUT) + .setNullInterpretation(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE)) + .setChild(RecordKeyExpressionProto.KeyExpression.newBuilder() + .setField(RecordKeyExpressionProto.Field.newBuilder() .setFieldName("name") - .setFanType(RecordMetaDataProto.Field.FanType.SCALAR) - .setNullInterpretation(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE)))))) + .setFanType(RecordKeyExpressionProto.Field.FanType.SCALAR) + .setNullInterpretation(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE)))))) .build(); Assertions.assertEquals(expected2, NullableArrayUtils.wrapArray(original2, parentDescriptor, true)); } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index ba8e3ec70a..743bac161f 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -879,18 +879,18 @@ void aliasingTableToResolveAmbiguityWorks() throws Exception { // (TODO): more tests: 1) invalid param type lat(x loc) 2) return (x+1) 3) nested functions @Test void testUserDefinedFunction() throws Exception { + // "name" is a reserved keyword, users need to put a double quote around it final String schemaTemplate = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n" + - "CREATE FUNCTION name(x Location) RETURNS string AS x.name\n"; + "CREATE FUNCTION \"name\"(x Location) RETURNS string AS x.name\n"; try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { try (var s = ddl.setSchemaAndGetConnection().createStatement()) { insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); } - // (TODO): double quote around loc, to differentiate udf from scalar function call, better way of doing it? - try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE name((loc)) = ?name")) { + try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE \"name\"(loc) = ?name")) { ps.setString("name", "Apple Park Visitor Center"); try (final RelationalResultSet resultSet = ps.executeQuery()) { ResultSetAssert.assertThat(resultSet) @@ -929,14 +929,14 @@ void testUserDefinedFunction3() throws Exception { "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n" + "CREATE FUNCTION name(x Location) RETURNS string AS x.name\n" + - "CREATE FUNCTION id(x uid) RETURNS bigint AS x"; + "CREATE FUNCTION id(x bigint) RETURNS bigint AS x"; try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { try (var s = ddl.setSchemaAndGetConnection().createStatement()) { insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); } // (TODO): double quote around loc, to differentiate udf from scalar function call, better way of doing it? - try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE id((uid)) = ?id")) { + try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE id(uid) = ?id")) { ps.setLong("id", 1L); try (final RelationalResultSet resultSet = ps.executeQuery()) { ResultSetAssert.assertThat(resultSet) From df4235ea38f330976179a36e6166c83345edf6f5 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 19 Dec 2024 17:08:04 -0800 Subject: [PATCH 24/41] clean --- .../src/main/antlr/RelationalLexer.g4 | 2 -- .../metadata/RecordLayerSchemaTemplate.java | 1 - .../UserDefinedFunctionDefinition.java | 11 ++++----- .../recordlayer/query/AstNormalizer.java | 1 - .../recordlayer/query/Identifier.java | 6 ++--- .../recordlayer/query/IndexGenerator.java | 6 ++--- .../recordlayer/query/SemanticAnalyzer.java | 24 +++++++++++-------- .../query/functions/SqlFunctionCatalog.java | 3 ++- .../query/visitors/DdlVisitor.java | 14 +++++++---- .../query/visitors/ExpressionVisitor.java | 4 +--- .../src/main/proto/continuation.proto | 1 - 11 files changed, 38 insertions(+), 35 deletions(-) diff --git a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 index 993bbeb4ff..682011f51f 100644 --- a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 @@ -647,7 +647,6 @@ RESUME: 'RESUME'; RETURNED_SQLSTATE: 'RETURNED_SQLSTATE'; RETURNING: 'RETURNING'; RETURNS: 'RETURNS'; -RIGHTARG: 'RIGHTARG'; ROLE: 'ROLE'; ROLLBACK: 'ROLLBACK'; ROLLUP: 'ROLLUP'; @@ -712,7 +711,6 @@ TRANSACTIONAL: 'TRANSACTIONAL'; TRIGGERS: 'TRIGGERS'; TRUNCATE: 'TRUNCATE'; TYPE: 'TYPE'; -UDF_: 'UDF_'; UNBOUNDED: 'UNBOUNDED'; UNDEFINED: 'UNDEFINED'; UNDOFILE: 'UNDOFILE'; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java index 27db340fbd..eab8884a8e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java @@ -50,7 +50,6 @@ import com.google.protobuf.Descriptors; import javax.annotation.Nonnull; -import javax.annotation.Nullable; import java.util.BitSet; import java.util.Collection; import java.util.HashSet; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java index 998b26694e..d4bf6309d9 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java @@ -27,7 +27,6 @@ import com.apple.foundationdb.record.query.plan.cascades.values.Value; import javax.annotation.Nonnull; -import java.util.ArrayList; import java.util.List; public class UserDefinedFunctionDefinition { @@ -44,17 +43,17 @@ public String getName() { return functionName; } + @Nonnull public BuiltInFunction getBuiltInFunction() { - return new BuiltInFunction(functionName, List.of(), (builtInFunction, arguments) -> { - final var result = ((MacroFunctionValue) functionValue).call(arguments); - return result; - }) {}; + return new BuiltInFunction(functionName, List.of(), (builtInFunction, arguments) -> ((MacroFunctionValue) functionValue).call(arguments)) {}; } - public Udf toUDF() { + @Nonnull + public Udf toUdf() { return new Udf(functionName, functionValue); } + @Nonnull public static UserDefinedFunctionDefinition fromUdf(Udf udf) { return new UserDefinedFunctionDefinition(udf.getFunctionName(), udf.getValue()); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java index 98bbdb9a55..f89278b3fb 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/AstNormalizer.java @@ -351,7 +351,6 @@ public Void visitContinuationAtom(@Nonnull RelationalParser.ContinuationAtomCont @Override @SuppressWarnings("PMD.CompareObjectsWithEquals") // deliberate use of pointer equality public Void visitScalarFunctionCall(@Nonnull RelationalParser.ScalarFunctionCallContext ctx) { - // final var functionName = ctx.scalarFunctionName() == null ? ctx.userDefinedFunctionName.getText() : ctx.scalarFunctionName().getText(); final var functionName = ctx.scalarFunctionName().getText(); boolean skipFirstFunctionArgument = "JAVA_CALL".equals(SemanticAnalyzer.normalizeString(functionName, false)); for (int i = 0; i < ctx.getChildCount(); i++) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java index 26b9c21347..f4b5b50eba 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/Identifier.java @@ -129,10 +129,10 @@ public boolean prefixedWith(@Nonnull Identifier identifier) { } @Nonnull - public Identifier removePrefix(@Nonnull Identifier identifier) { + public List removePrefix(@Nonnull Identifier prefix) { + // assume the identifier has the prefix, should call prefixedWith(prefix) to check before calling this method final var fullName = fullyQualifiedName(); - final var newIdFullName = fullName.subList(identifier.fullyQualifiedName().size(), fullName.size()); - return Identifier.of(newIdFullName.get(newIdFullName.size() - 1), newIdFullName.subList(0, newIdFullName.size() - 1)); + return fullName.subList(prefix.fullyQualifiedName().size(), fullName.size()); } public boolean qualifiedWith(@Nonnull Identifier identifier) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/IndexGenerator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/IndexGenerator.java index c2b412d885..52e93e663d 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/IndexGenerator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/IndexGenerator.java @@ -517,7 +517,7 @@ private KeyExpression toKeyExpression(Value value, Map orderingFu } @Nonnull - public static KeyExpression toKeyExpression(@Nonnull Value value) { + private KeyExpression toKeyExpression(@Nonnull Value value) { if (value instanceof VersionValue) { return VersionKeyExpression.VERSION; } else if (value instanceof FieldValue) { @@ -732,12 +732,12 @@ private Value dereference(@Nonnull Value value) { } @Nonnull - private static KeyExpression toKeyExpression(@Nonnull List> fields) { + private KeyExpression toKeyExpression(@Nonnull List> fields) { return toKeyExpression(fields, 0); } @Nonnull - private static KeyExpression toKeyExpression(@Nonnull List> fields, int index) { + private KeyExpression toKeyExpression(@Nonnull List> fields, int index) { Assert.thatUnchecked(!fields.isEmpty()); final var field = fields.get(index); final var keyExpression = toKeyExpression(field.getLeft(), field.getRight()); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index f5513529cb..c61175cb5d 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -461,25 +461,28 @@ public Optional lookUpNestedField(@Nonnull Identifier requestedIdent @Nonnull public Optional lookUpNestedField(@Nonnull Identifier requestedIdentifier, @Nonnull Identifier paramId, - @Nonnull QuantifiedObjectValue objectValue) { + @Nonnull QuantifiedObjectValue existingValue, + @Nonnull DataType targetDataType) { Assert.thatUnchecked(requestedIdentifier.prefixedWith(paramId), "Invalid function definition"); + // x -> x if (requestedIdentifier.fullyQualifiedName().size() == paramId.fullyQualifiedName().size()) { - return Optional.of(objectValue); + return Optional.of(existingValue); } - final var remainingPath = requestedIdentifier.removePrefix(paramId).fullyQualifiedName(); + // find nested field path + final var remainingPath = requestedIdentifier.removePrefix(paramId); final ImmutableList.Builder accessors = ImmutableList.builder(); - DataType currentDataType = DataTypeUtils.toRelationalType(objectValue.getResultType()); + DataType existingDataType = DataTypeUtils.toRelationalType(existingValue.getResultType()); for (String s : remainingPath) { - if (currentDataType.getCode() != DataType.Code.STRUCT) { + if (existingDataType.getCode() != DataType.Code.STRUCT) { return Optional.empty(); } - final var fields = ((DataType.StructType) currentDataType).getFields(); + final var fields = ((DataType.StructType) existingDataType).getFields(); var found = false; for (int j = 0; j < fields.size(); j++) { if (fields.get(j).getName().equals(s)) { accessors.add(new FieldValue.Accessor(fields.get(j).getName(), j)); - currentDataType = fields.get(j).getType(); + existingDataType = fields.get(j).getType(); found = true; break; } @@ -489,9 +492,10 @@ public Optional lookUpNestedField(@Nonnull Identifier requestedIdentifier } } // probably need to check if currentDataType = targetDataType - final var fieldPath = FieldValue.resolveFieldPath(objectValue.getResultType(), accessors.build()); - final var attributeExpression = FieldValue.ofFieldsAndFuseIfPossible(objectValue, fieldPath); - return Optional.of(attributeExpression); + final var fieldPath = FieldValue.resolveFieldPath(existingValue.getResultType(), accessors.build()); + final var fieldValue = FieldValue.ofFieldsAndFuseIfPossible(existingValue, fieldPath); + Assert.thatUnchecked(fieldValue.getResultType().equals(DataTypeUtils.toRecordLayerType(targetDataType)), ErrorCode.DATATYPE_MISMATCH, "Result data types don't match!"); + return Optional.of(fieldValue); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java index 959a9e500a..a618dbdd8a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java @@ -115,7 +115,8 @@ public static SqlFunctionCatalog instance() { return INSTANCE; } - public BuiltInFunction addFunction(UserDefinedFunctionDefinition functionDefinition) { + @Nonnull + public BuiltInFunction addFunction(@Nonnull UserDefinedFunctionDefinition functionDefinition) { final var func = functionDefinition.getBuiltInFunction(); udfMap.put(functionDefinition.getName(), func); return func; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index ce258ca231..988615d5ae 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -53,6 +53,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.Objects; import java.util.Optional; @API(API.Status.EXPERIMENTAL) @@ -187,14 +188,19 @@ public UserDefinedFunctionDefinition visitFunctionDefinition(@Nonnull Relational getDelegate().replaceCatalog(ddlCatalog); final var semanticAnalyzer = getDelegate().getSemanticAnalyzer(); + // argumentValue final var inputColumnId = ctx.columnType(0).customType != null ? visitUid(ctx.columnType(0).customType) : Identifier.of(ctx.columnType(0).getText()); final var columnType = semanticAnalyzer.lookupType(inputColumnId, true, false, metadataBuilder::findType); - QuantifiedObjectValue argumentValue = QuantifiedObjectValue.of(CorrelationIdentifier.of(ctx.paramName.getText()), DataTypeUtils.toRecordLayerType(columnType)); + QuantifiedObjectValue argumentValue = QuantifiedObjectValue.of(CorrelationIdentifier.uniqueID(), DataTypeUtils.toRecordLayerType(columnType)); - final var id = visitFullId(ctx.fullId()); + // function return type + final var returnColumnId = ctx.columnType(1).customType != null ? visitUid(ctx.columnType(1).customType) : Identifier.of(ctx.columnType(1).getText()); + final var returnType = semanticAnalyzer.lookupType(returnColumnId, true, false, metadataBuilder::findType); + + final var functionBody = visitFullId(ctx.fullId()); final var paramNameId = Identifier.of(ctx.paramName.getText().toUpperCase(Locale.ROOT)); - Optional fieldValue = semanticAnalyzer.lookUpNestedField(id, paramNameId, argumentValue); + Optional fieldValue = semanticAnalyzer.lookUpNestedField(functionBody, paramNameId, argumentValue, returnType); Assert.thatUnchecked(fieldValue.isPresent(), "couldn't resolve function definition"); return new UserDefinedFunctionDefinition(ctx.functionName.getText(), MacroFunctionValue.of(List.of(argumentValue), fieldValue.get())); @@ -260,7 +266,7 @@ public ProceduralPlan visitCreateSchemaTemplateStatement(@Nonnull RelationalPars metadataBuilder.addTable(tableWithIndex); } final var functions = functionClauses.build().stream().map(this::visitFunctionDefinition).collect(ImmutableList.toImmutableList()); - final var udfs = functions.stream().map(v -> v.toUDF()).collect(Collectors.toList()); + final var udfs = functions.stream().map(v -> v.toUdf()).collect(Collectors.toList()); metadataBuilder.addUdfs(udfs); return ProceduralPlan.of(metadataOperationsFactory.getCreateSchemaTemplateConstantAction(metadataBuilder.build(), Options.NONE)); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index cb4d6752b9..f0a81c044d 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -255,9 +255,7 @@ public Expression visitCaseFunctionCall(@Nonnull RelationalParser.CaseFunctionCa @Override public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { final var functionName = ctx.userDefinedFunctionName().getText(); - // special case for user-defined functions where we want to exclude the first argument from - // being literal-stripped. - @Nonnull Expressions arguments = visitFunctionArgs(ctx.functionArgs()); + Expressions arguments = visitFunctionArgs(ctx.functionArgs()); return getDelegate().resolveFunction(functionName, arguments.asList().toArray(new Expression[0])); } diff --git a/fdb-relational-core/src/main/proto/continuation.proto b/fdb-relational-core/src/main/proto/continuation.proto index 4b5dfae0d3..470e9dc7e2 100644 --- a/fdb-relational-core/src/main/proto/continuation.proto +++ b/fdb-relational-core/src/main/proto/continuation.proto @@ -22,7 +22,6 @@ syntax = "proto3"; package com.apple.foundationdb.relational.continuation; import "record_metadata.proto"; import "record_query_plan.proto"; -import "record_query_runtime.proto"; option java_multiple_files = true; option java_package = "com.apple.foundationdb.relational.continuation"; From c343ed39538f335182855bbb9496aea06e103f81 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 19 Dec 2024 20:49:07 -0800 Subject: [PATCH 25/41] more test --- .../recordlayer/query/SemanticAnalyzer.java | 1 + .../recordlayer/query/StandardQueryTests.java | 76 ++++++++++++++----- 2 files changed, 58 insertions(+), 19 deletions(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index c61175cb5d..28b472535a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -467,6 +467,7 @@ public Optional lookUpNestedField(@Nonnull Identifier requestedIdentifier // x -> x if (requestedIdentifier.fullyQualifiedName().size() == paramId.fullyQualifiedName().size()) { + Assert.thatUnchecked(existingValue.getResultType().equals(DataTypeUtils.toRecordLayerType(targetDataType)), ErrorCode.DATATYPE_MISMATCH, "Result data types don't match!"); return Optional.of(existingValue); } // find nested field path diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index 743bac161f..162ba9efaa 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -31,6 +31,7 @@ import com.apple.foundationdb.relational.api.RelationalStructMetaData; import com.apple.foundationdb.relational.api.exceptions.ContextualSQLException; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.recordlayer.ContinuationImpl; import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalExtension; import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalStatement; @@ -51,6 +52,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URI; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.sql.Array; import java.sql.Connection; @@ -876,7 +878,6 @@ void aliasingTableToResolveAmbiguityWorks() throws Exception { } } - // (TODO): more tests: 1) invalid param type lat(x loc) 2) return (x+1) 3) nested functions @Test void testUserDefinedFunction() throws Exception { // "name" is a reserved keyword, users need to put a double quote around it @@ -884,7 +885,8 @@ void testUserDefinedFunction() throws Exception { "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n" + - "CREATE FUNCTION \"name\"(x Location) RETURNS string AS x.name\n"; + "CREATE FUNCTION \"name\"(x Location) RETURNS string AS x.name\n" + + "CREATE FUNCTION id(x bigint) RETURNS bigint AS x\n"; try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { try (var s = ddl.setSchemaAndGetConnection().createStatement()) { @@ -907,7 +909,9 @@ void testUserDefinedFunction2() throws Exception { final String schemaTemplate = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + - "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n"; + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n" + + "CREATE FUNCTION \"name\"(x Location) RETURNS string AS x.name\n" + + "CREATE FUNCTION id(x bigint) RETURNS bigint AS x\n"; try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { try (var s = ddl.setSchemaAndGetConnection().createStatement()) { @@ -935,7 +939,6 @@ void testUserDefinedFunction3() throws Exception { try (var s = ddl.setSchemaAndGetConnection().createStatement()) { insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); } - // (TODO): double quote around loc, to differentiate udf from scalar function call, better way of doing it? try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE id(uid) = ?id")) { ps.setLong("id", 1L); try (final RelationalResultSet resultSet = ps.executeQuery()) { @@ -948,21 +951,40 @@ void testUserDefinedFunction3() throws Exception { } } - private RelationalStruct insertLocationComplexRecord(RelationalStatement s, long uid, @Nonnull final String name, String latitude, String longitude) throws SQLException { - var coord = EmbeddedRelationalStruct.newBuilder() - .addString("LATITUDE", latitude) - .addString("LONGITUDE", longitude) - .build(); - var struct = EmbeddedRelationalStruct.newBuilder() - .addLong("UID", uid) - .addStruct("LOC", EmbeddedRelationalStruct.newBuilder() - .addString("NAME", name) - .addStruct("COORD", coord) - .build()) - .build(); - int cnt = s.executeInsert("T1", struct); - Assertions.assertEquals(1, cnt, "Incorrect insertion count"); - return struct; + @Test + void testIncorrectUserDefinedFunction() throws Exception { + final String schemaTemplate1 = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coor.latitude\n"; // "coor" is wrong + + // fail to build the MacroFunctionValue in the DDL step + final var errorMsg1 = Assertions.assertThrows(SQLException.class, () -> Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate1).build()).getMessage(); + Assertions.assertTrue(errorMsg1.contains("couldn't resolve function")); + + final String schemaTemplate2 = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + + "CREATE FUNCTION id(x bigint) RETURNS string AS x\n"; // wrong return type + // fail to build the MacroFunctionValue in the DDL step + final var errorMsg2 = Assertions.assertThrows(SQLException.class, () -> Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate2).build()).getMessage(); + Assertions.assertTrue(errorMsg2.contains("Result data types don't match")); + + final String schemaTemplate3 = "CREATE TYPE AS STRUCT LATLON (latitude string, longitude string)\n" + + "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + + "CREATE FUNCTION name(x Location) RETURNS string AS x.name\n"; // no double quote around name + + try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate3).build()) { + try (var s = ddl.setSchemaAndGetConnection().createStatement()) { + insertLocationComplexRecord(s, 1L, "Apple Park Visitor Center", "37.3", "-120.0"); + } + try (var ps = ddl.setSchemaAndGetConnection().prepareStatement("SELECT * FROM T1 WHERE name(loc) = ?name")) { + ps.setString("name", "Apple Park Visitor Center"); + final var errorMsg3 = Assertions.assertThrows(SQLException.class, ps::executeQuery).getMessage(); + Assertions.assertTrue(errorMsg3.contains("syntax error")); + } + } } @Test @@ -1615,4 +1637,20 @@ private RelationalStruct insertRestaurantComplexRecord(RelationalStatement s, in Assertions.assertEquals(1, cnt, "Incorrect insertion count"); return struct; } + + private void insertLocationComplexRecord(@Nonnull RelationalStatement s, long uid, @Nonnull final String name, @Nonnull String latitude, @Nonnull String longitude) throws SQLException { + var coord = EmbeddedRelationalStruct.newBuilder() + .addString("LATITUDE", latitude) + .addString("LONGITUDE", longitude) + .build(); + var struct = EmbeddedRelationalStruct.newBuilder() + .addLong("UID", uid) + .addStruct("LOC", EmbeddedRelationalStruct.newBuilder() + .addString("NAME", name) + .addStruct("COORD", coord) + .build()) + .build(); + int cnt = s.executeInsert("T1", struct); + Assertions.assertEquals(1, cnt, "Incorrect insertion count"); + } } From 6aaae1da711185cebf110d14af8cac70b6b7bcd7 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Fri, 10 Jan 2025 09:50:14 -0800 Subject: [PATCH 26/41] save --- .../UserDefinedFunctionDefinition.java | 60 ------------------- .../recordlayer/query/SemanticAnalyzer.java | 3 +- .../recordlayer/query/TautologicalValue.java | 6 ++ .../query/functions/FunctionCatalog.java | 4 +- .../query/functions/SqlFunctionCatalog.java | 19 ++++-- .../query/visitors/DdlVisitor.java | 13 ++-- 6 files changed, 28 insertions(+), 77 deletions(-) delete mode 100644 fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java deleted file mode 100644 index d4bf6309d9..0000000000 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/UserDefinedFunctionDefinition.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * UserDefinedFunctionDefinition.java - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apple.foundationdb.relational.recordlayer.metadata; - -import com.apple.foundationdb.record.metadata.Udf; -import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; -import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; -import com.apple.foundationdb.record.query.plan.cascades.values.MacroFunctionValue; -import com.apple.foundationdb.record.query.plan.cascades.values.Value; - -import javax.annotation.Nonnull; -import java.util.List; - -public class UserDefinedFunctionDefinition { - @Nonnull private final String functionName; - @Nonnull private final Value functionValue; - - public UserDefinedFunctionDefinition(@Nonnull String functionName, @Nonnull Value value) { - this.functionName = functionName; - this.functionValue = value; - } - - @Nonnull - public String getName() { - return functionName; - } - - @Nonnull - public BuiltInFunction getBuiltInFunction() { - return new BuiltInFunction(functionName, List.of(), (builtInFunction, arguments) -> ((MacroFunctionValue) functionValue).call(arguments)) {}; - } - - @Nonnull - public Udf toUdf() { - return new Udf(functionName, functionValue); - } - - @Nonnull - public static UserDefinedFunctionDefinition fromUdf(Udf udf) { - return new UserDefinedFunctionDefinition(udf.getFunctionName(), udf.getValue()); - } -} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index 28b472535a..57d429d07e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -38,6 +38,7 @@ import com.apple.foundationdb.relational.api.metadata.SchemaTemplate; import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.query.functions.FunctionCatalog; import com.apple.foundationdb.relational.recordlayer.query.functions.SqlFunctionCatalog; import com.apple.foundationdb.relational.util.Assert; @@ -84,7 +85,7 @@ public SemanticAnalyzer(@Nonnull SchemaTemplate metadataCatalog, this.functionCatalog = functionCatalog; // add UDFs to functionCatalog for (Udf udf: ((RecordLayerSchemaTemplate) metadataCatalog).getAllUdfs()) { - ((SqlFunctionCatalog)this.functionCatalog).addFunction(UserDefinedFunctionDefinition.fromUdf(udf)); + ((SqlFunctionCatalog)this.functionCatalog).addFunction(udf); } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/TautologicalValue.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/TautologicalValue.java index cde352e254..56b250adb5 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/TautologicalValue.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/TautologicalValue.java @@ -82,6 +82,12 @@ protected Iterable computeChildren() { return ImmutableList.of(); } + @Nonnull + @Override + public ExplainTokensWithPrecedence explain(@Nonnull Iterable> iterable) { + return null; + } + @Nullable @Override public Object eval(@Nonnull FDBRecordStoreBase store, @Nonnull EvaluationContext context) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/FunctionCatalog.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/FunctionCatalog.java index b6f13ea996..2b7c26d746 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/FunctionCatalog.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/FunctionCatalog.java @@ -20,14 +20,14 @@ package com.apple.foundationdb.relational.recordlayer.query.functions; -import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; +import com.apple.foundationdb.record.query.plan.cascades.Function; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; import javax.annotation.Nonnull; public interface FunctionCatalog { @Nonnull - BuiltInFunction lookUpFunction(@Nonnull String name); + Function lookUpFunction(@Nonnull String name); boolean containsFunction(@Nonnull String name); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java index a618dbdd8a..6df27090b9 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java @@ -21,8 +21,9 @@ package com.apple.foundationdb.relational.recordlayer.query.functions; import com.apple.foundationdb.annotation.API; - +import com.apple.foundationdb.record.metadata.Udf; import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; +import com.apple.foundationdb.record.query.plan.cascades.Function; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; @@ -51,14 +52,20 @@ public final class SqlFunctionCatalog implements FunctionCatalog { @Nonnull private final ImmutableMap> synonyms; + @Nonnull + private final Map> udfMap; + private SqlFunctionCatalog() { this.synonyms = createSynonyms(); } @Nonnull @Override - public BuiltInFunction lookUpFunction(@Nonnull String name) { - return Assert.notNullUnchecked(synonyms.get(name.toLowerCase(Locale.ROOT))); + public Function lookUpFunction(@Nonnull String name) { + if (synonyms.get(name.toLowerCase(Locale.ROOT)) != null) { + return Objects.requireNonNull(synonyms.get(name.toLowerCase(Locale.ROOT))); + } + return Assert.notNullUnchecked(udfMap.get(name.toLowerCase(Locale.ROOT))); } @Override @@ -116,9 +123,9 @@ public static SqlFunctionCatalog instance() { } @Nonnull - public BuiltInFunction addFunction(@Nonnull UserDefinedFunctionDefinition functionDefinition) { - final var func = functionDefinition.getBuiltInFunction(); - udfMap.put(functionDefinition.getName(), func); + public Function addFunction(@Nonnull Udf udf) { + final var func = udf.getMacroFunction(); + udfMap.put(udf.getFunctionName(), func); return func; } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 988615d5ae..6a8d69c658 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -21,10 +21,10 @@ package com.apple.foundationdb.relational.recordlayer.query.visitors; import com.apple.foundationdb.annotation.API; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.metadata.Udf; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.MacroFunction; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalSortExpression; -import com.apple.foundationdb.record.query.plan.cascades.values.MacroFunctionValue; import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.relational.api.Options; @@ -37,7 +37,6 @@ import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; -import com.apple.foundationdb.relational.recordlayer.metadata.UserDefinedFunctionDefinition; import com.apple.foundationdb.relational.recordlayer.query.Identifier; import com.apple.foundationdb.relational.recordlayer.query.IndexGenerator; import com.apple.foundationdb.relational.recordlayer.query.LogicalOperator; @@ -53,7 +52,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; -import java.util.Objects; import java.util.Optional; @API(API.Status.EXPERIMENTAL) @@ -182,7 +180,7 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi @Nonnull @Override - public UserDefinedFunctionDefinition visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { + public Udf visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { final var ddlCatalog = metadataBuilder.build(); // parse the function definition using the newly constructed metadata. getDelegate().replaceCatalog(ddlCatalog); @@ -203,7 +201,7 @@ public UserDefinedFunctionDefinition visitFunctionDefinition(@Nonnull Relational Optional fieldValue = semanticAnalyzer.lookUpNestedField(functionBody, paramNameId, argumentValue, returnType); Assert.thatUnchecked(fieldValue.isPresent(), "couldn't resolve function definition"); - return new UserDefinedFunctionDefinition(ctx.functionName.getText(), MacroFunctionValue.of(List.of(argumentValue), fieldValue.get())); + return new Udf(new MacroFunction(ctx.functionName.getText(), List.of(argumentValue.getResultType()), List.of(argumentValue.getAlias()), fieldValue.get())); } @Nonnull @@ -265,8 +263,7 @@ public ProceduralPlan visitCreateSchemaTemplateStatement(@Nonnull RelationalPars final var tableWithIndex = RecordLayerTable.Builder.from(table).addIndex(index).build(); metadataBuilder.addTable(tableWithIndex); } - final var functions = functionClauses.build().stream().map(this::visitFunctionDefinition).collect(ImmutableList.toImmutableList()); - final var udfs = functions.stream().map(v -> v.toUdf()).collect(Collectors.toList()); + final var udfs = functionClauses.build().stream().map(this::visitFunctionDefinition).collect(ImmutableList.toImmutableList()); metadataBuilder.addUdfs(udfs); return ProceduralPlan.of(metadataOperationsFactory.getCreateSchemaTemplateConstantAction(metadataBuilder.build(), Options.NONE)); } From 4f2b546ce0296b16a898e50b9b83bec3f9b58c2f Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Fri, 10 Jan 2025 12:49:08 -0800 Subject: [PATCH 27/41] move code --- .idea/compiler.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.idea/compiler.xml b/.idea/compiler.xml index 696ba65d1a..4cda564c0c 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -63,15 +63,7 @@ - - - - - - - - - + @@ -86,6 +78,14 @@ + + + + + + + + From e11b3159ad3e51dd4fa5a51b9c2825f7a78d7152 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Fri, 10 Jan 2025 18:00:48 -0800 Subject: [PATCH 28/41] add test --- .../foundationdb/record/metadata/Udf.java | 4 +- .../query/plan/cascades/BuiltInFunction.java | 3 +- .../record/query/plan/cascades/Function.java | 2 +- .../query/plan/cascades/MacroFunction.java | 12 +-- .../plan/cascades/MacroFunctionTest.java | 78 +++++++++++++++++++ 5 files changed, 90 insertions(+), 9 deletions(-) create mode 100644 fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunctionTest.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java index b089be2dba..01644fbee4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java @@ -45,7 +45,9 @@ public MacroFunction getMacroFunction() { } @Nonnull - public String getFunctionName() {return macroFunction.getFunctionName();} + public String getFunctionName() { + return macroFunction.getFunctionName(); + } @Nonnull public RecordMetaDataProto.Udf toProto() { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java index f604752716..e3a48bd1a9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java @@ -38,9 +38,10 @@ * @param The resulting type of the function. */ @SuppressWarnings("PMD.AbstractClassWithoutAbstractMethod") -public abstract class BuiltInFunction extends Function{ +public abstract class BuiltInFunction extends Function { @Nonnull private final EncapsulationFunction encapsulationFunction; + /** * Creates a new instance of {@link BuiltInFunction}. * @param functionName The name of the function. diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java index cde10e2d17..cb4db3e013 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java @@ -35,7 +35,7 @@ * @param The resulting type of the function. */ @SuppressWarnings("PMD.AbstractClassWithoutAbstractMethod") -public abstract class Function{ +public abstract class Function { @Nonnull final String functionName; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java index bb7c69ee38..728404cf8d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java @@ -37,14 +37,14 @@ /** * Main interface for defining a user-defined function that can be evaluated against a number of arguments. */ -public class MacroFunction extends Function{ +public class MacroFunction extends Function { @Nonnull private final com.apple.foundationdb.record.query.plan.cascades.values.Value bodyValue; private final List parameterIdentifiers; - public MacroFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final List parameterIdentifiers, @Nonnull final com.apple.foundationdb.record.query.plan.cascades.values.Value bodyValue) { - super(functionName, parameterTypes, null); - this.parameterIdentifiers = ImmutableList.copyOf(parameterIdentifiers); + public MacroFunction(@Nonnull final String functionName, @Nonnull final List parameters, @Nonnull final com.apple.foundationdb.record.query.plan.cascades.values.Value bodyValue) { + super(functionName, parameters.stream().map(QuantifiedObjectValue::getResultType).collect(Collectors.toList()), null); + this.parameterIdentifiers = parameters.stream().map(QuantifiedObjectValue::getAlias).collect(Collectors.toList()); this.bodyValue = bodyValue; } @@ -57,6 +57,7 @@ public com.apple.foundationdb.record.query.plan.cascades.values.Value encapsulat for (int i = 0; i < arguments.size(); i++) { // check that arguments[i] type matches with parameterTypes[i] final int finalI = i; + System.out.println("argument[i]:" + arguments.get(finalI).getResultType() + " class:" + arguments.get(finalI).getResultType().getClass() + " parameterTYpes:" + parameterTypes.get(i)); SemanticException.check(arguments.get(finalI).getResultType().equals(parameterTypes.get(i)), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument type doesn't match with function definition"); translationMapBuilder.when(parameterIdentifiers.get(finalI)).then((sourceAlias, leafValue) -> (Value)arguments.get(finalI)); } @@ -85,8 +86,7 @@ public PValue toValueProto(@Nonnull final PlanSerializationContext serialization @Nonnull public static MacroFunction fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PMacroFunctionValue functionValue) { return new MacroFunction(functionValue.getFunctionName(), - functionValue.getArgumentsList().stream().map(pvalue -> Value.fromValueProto(serializationContext, pvalue).getResultType()).collect(Collectors.toList()), - functionValue.getArgumentsList().stream().map(pvalue -> ((QuantifiedObjectValue)Value.fromValueProto(serializationContext, pvalue)).getAlias()).collect(Collectors.toList()), + functionValue.getArgumentsList().stream().map(pvalue -> ((QuantifiedObjectValue)Value.fromValueProto(serializationContext, pvalue))).collect(Collectors.toList()), Value.fromValueProto(serializationContext, functionValue.getBody())); } } diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunctionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunctionTest.java new file mode 100644 index 0000000000..aac335b98f --- /dev/null +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunctionTest.java @@ -0,0 +1,78 @@ +/* + * MacroFunctionTest.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades; + + +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.ArithmeticValue; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; +import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; +import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; +import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; + +/** + * Tests of {@link MacroFunction}. + */ +public class MacroFunctionTest { + @Test + void testColumnProjection() { + List fields = List.of( + Type.Record.Field.of(Type.primitiveType(Type.TypeCode.STRING), Optional.of("name")), + Type.Record.Field.of(Type.primitiveType(Type.TypeCode.LONG), Optional.of("id"))); + Type record = Type.Record.fromFields(false, fields); + QuantifiedObjectValue param = QuantifiedObjectValue.of(CorrelationIdentifier.uniqueID(), record); + FieldValue bodyValue = FieldValue.ofFieldName(param, "name"); + MacroFunction macroFunction = new MacroFunction("getName", List.of(param), bodyValue); + + RecordConstructorValue testValue1 = RecordConstructorValue.ofColumns(List.of( + Column.of(fields.get(0), new LiteralValue<>(fields.get(0).getFieldType(), "Rose")), + Column.of(fields.get(1), new LiteralValue<>(fields.get(1).getFieldType(), 1L)) + )); + + Assertions.assertEquals(FieldValue.ofFieldName(testValue1, "name"), macroFunction.encapsulate(List.of(testValue1))); + } + + @Test + void testAdd() { + List fields = List.of( + Type.Record.Field.of(Type.primitiveType(Type.TypeCode.LONG), Optional.of("id"))); + Type record = Type.Record.fromFields(false, fields); + QuantifiedObjectValue param1 = QuantifiedObjectValue.of(CorrelationIdentifier.uniqueID(), record); + QuantifiedObjectValue param2 = QuantifiedObjectValue.of(CorrelationIdentifier.uniqueID(), record); + + ArithmeticValue bodyValue = new ArithmeticValue(ArithmeticValue.PhysicalOperator.ADD_LL, param1, param2); + MacroFunction macroFunction = new MacroFunction("add", List.of(param1, param2), bodyValue); + + RecordConstructorValue testValue1 = RecordConstructorValue.ofColumns(List.of( + Column.of(fields.get(0), new LiteralValue<>(fields.get(0).getFieldType(), 1L)) + )); + RecordConstructorValue testValue2 = RecordConstructorValue.ofColumns(List.of( + Column.of(fields.get(0), new LiteralValue<>(fields.get(0).getFieldType(), 2L)) + )); + + Assertions.assertEquals(new ArithmeticValue(ArithmeticValue.PhysicalOperator.ADD_LL, testValue1, testValue2), macroFunction.encapsulate(List.of(testValue1, testValue2))); + } +} From 29b9cb2a4bad16329de86ab17af66439d73647c4 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Sun, 12 Jan 2025 23:48:34 -0800 Subject: [PATCH 29/41] pmd --- .../record/query/plan/cascades/Function.java | 2 +- .../record/query/plan/cascades/MacroFunction.java | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java index cb4db3e013..131de81195 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java @@ -103,7 +103,7 @@ public boolean hasVariadicSuffix() { } @Nonnull - public abstract T encapsulate(@Nonnull final List arguments); + public abstract T encapsulate(@Nonnull List arguments); @Nonnull @Override diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java index 728404cf8d..b4689258df 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunction.java @@ -23,12 +23,10 @@ import com.apple.foundationdb.record.PlanSerializationContext; import com.apple.foundationdb.record.planprotos.PMacroFunctionValue; import com.apple.foundationdb.record.planprotos.PValue; -import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; -import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; import java.util.List; @@ -37,12 +35,12 @@ /** * Main interface for defining a user-defined function that can be evaluated against a number of arguments. */ -public class MacroFunction extends Function { +public class MacroFunction extends Function { @Nonnull - private final com.apple.foundationdb.record.query.plan.cascades.values.Value bodyValue; + private final Value bodyValue; private final List parameterIdentifiers; - public MacroFunction(@Nonnull final String functionName, @Nonnull final List parameters, @Nonnull final com.apple.foundationdb.record.query.plan.cascades.values.Value bodyValue) { + public MacroFunction(@Nonnull final String functionName, @Nonnull final List parameters, @Nonnull final Value bodyValue) { super(functionName, parameters.stream().map(QuantifiedObjectValue::getResultType).collect(Collectors.toList()), null); this.parameterIdentifiers = parameters.stream().map(QuantifiedObjectValue::getAlias).collect(Collectors.toList()); this.bodyValue = bodyValue; @@ -50,14 +48,13 @@ public MacroFunction(@Nonnull final String functionName, @Nonnull final List arguments) { + public Value encapsulate(@Nonnull List arguments) { // replace the QuantifiedObjectValue in body with arguments SemanticException.check(arguments.size() == parameterTypes.size(), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument length doesn't match with function definition"); TranslationMap.Builder translationMapBuilder = new TranslationMap.Builder(); for (int i = 0; i < arguments.size(); i++) { // check that arguments[i] type matches with parameterTypes[i] final int finalI = i; - System.out.println("argument[i]:" + arguments.get(finalI).getResultType() + " class:" + arguments.get(finalI).getResultType().getClass() + " parameterTYpes:" + parameterTypes.get(i)); SemanticException.check(arguments.get(finalI).getResultType().equals(parameterTypes.get(i)), SemanticException.ErrorCode.FUNCTION_UNDEFINED_FOR_GIVEN_ARGUMENT_TYPES, "argument type doesn't match with function definition"); translationMapBuilder.when(parameterIdentifiers.get(finalI)).then((sourceAlias, leafValue) -> (Value)arguments.get(finalI)); } From 52b3ba4274c8ff16826a08d56e72ede7ae2fa8bd Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Tue, 14 Jan 2025 14:56:17 -0800 Subject: [PATCH 30/41] comments --- .../foundationdb/record/RecordMetaData.java | 16 +++++++-------- .../record/RecordMetaDataBuilder.java | 20 +++++++++---------- .../{Udf.java => ScalarValuedFunction.java} | 14 ++++++------- .../src/main/proto/record_metadata.proto | 4 ++-- 4 files changed, 27 insertions(+), 27 deletions(-) rename fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/{Udf.java => ScalarValuedFunction.java} (79%) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java index 81ea4e218e..cd46adbc40 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java @@ -28,7 +28,7 @@ import com.apple.foundationdb.record.metadata.MetaDataException; import com.apple.foundationdb.record.metadata.RecordType; import com.apple.foundationdb.record.metadata.SyntheticRecordType; -import com.apple.foundationdb.record.metadata.Udf; +import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.metadata.UnnestedRecordType; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression; @@ -85,7 +85,7 @@ public class RecordMetaData implements RecordMetaDataProvider { @Nonnull private final Map> recordTypeKeyToSyntheticTypeMap; @Nonnull - private final Set udfs; + private final Set scalarValuedFunctions; @Nonnull private final Map indexes; @Nonnull @@ -116,7 +116,7 @@ protected RecordMetaData(@Nonnull RecordMetaData orig) { Collections.unmodifiableMap(orig.indexes), Collections.unmodifiableMap(orig.universalIndexes), Collections.unmodifiableList(orig.formerIndexes), - Collections.unmodifiableSet(orig.udfs), + Collections.unmodifiableSet(orig.scalarValuedFunctions), orig.splitLongRecords, orig.storeRecordVersions, orig.version, @@ -136,7 +136,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor, @Nonnull Map indexes, @Nonnull Map universalIndexes, @Nonnull List formerIndexes, - @Nonnull Set udfs, + @Nonnull Set scalarValuedFunctions, boolean splitLongRecords, boolean storeRecordVersions, int version, @@ -153,7 +153,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor, this.indexes = indexes; this.universalIndexes = universalIndexes; this.formerIndexes = formerIndexes; - this.udfs = udfs; + this.scalarValuedFunctions = scalarValuedFunctions; this.splitLongRecords = splitLongRecords; this.storeRecordVersions = storeRecordVersions; this.version = version; @@ -350,8 +350,8 @@ public List getFormerIndexes() { } @Nonnull - public Collection getAllUdfs() { - return udfs; + public Collection getAllScalarValuedFunctions() { + return scalarValuedFunctions; } public boolean isSplitLongRecords() { @@ -705,7 +705,7 @@ public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor } // Add in the final options. - builder.addAllUdfs(udfs.stream().map(Udf::toProto).collect(Collectors.toList())); + builder.addAllScalarValuedFunction(scalarValuedFunctions.stream().map(ScalarValuedFunction::toProto).collect(Collectors.toList())); builder.setSplitLongRecords(splitLongRecords); builder.setStoreRecordVersions(storeRecordVersions); builder.setVersion(version); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index 1c47cdcb97..48dfbb7f9c 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -35,7 +35,7 @@ import com.apple.foundationdb.record.metadata.RecordTypeIndexesBuilder; import com.apple.foundationdb.record.metadata.SyntheticRecordType; import com.apple.foundationdb.record.metadata.SyntheticRecordTypeBuilder; -import com.apple.foundationdb.record.metadata.Udf; +import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.metadata.UnnestedRecordTypeBuilder; import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; @@ -111,7 +111,7 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider { @Nonnull private final Map> syntheticRecordTypes; @Nonnull - private final Set udfs; + private final Set scalarValuedFunctions; @Nonnull private final Map indexes; @Nonnull @@ -147,7 +147,7 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider { indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance(); evolutionValidator = MetaDataEvolutionValidator.getDefaultInstance(); syntheticRecordTypes = new HashMap<>(); - udfs = new HashSet<>(); + scalarValuedFunctions = new HashSet<>(); } private void processSchemaOptions(boolean processExtensionOptions) { @@ -226,8 +226,8 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey())); } } - for (RecordMetaDataProto.Udf udf: metaDataProto.getUdfsList()) { - udfs.add(Udf.fromProto(udf)); + for (RecordMetaDataProto.ScalarValuedFunction scalarValuedFunction: metaDataProto.getScalarValuedFunctionList()) { + scalarValuedFunctions.add(ScalarValuedFunction.fromProto(scalarValuedFunction)); } if (metaDataProto.hasSplitLongRecords()) { splitLongRecords = metaDataProto.getSplitLongRecords(); @@ -1186,12 +1186,12 @@ public void addFormerIndex(@Nonnull FormerIndex formerIndex) { formerIndexes.add(formerIndex); } - public void addUdf(@Nonnull Udf udf) { - udfs.add(udf); + public void addScalarValuedFunction(@Nonnull ScalarValuedFunction scalarValuedFunction) { + scalarValuedFunctions.add(scalarValuedFunction); } - public void addUdfs(@Nonnull Iterable udfs) { - udfs.forEach(udf -> this.udfs.add(udf)); + public void addScalarValuedFunctions(@Nonnull Iterable scalarValuedFunctions) { + scalarValuedFunctions.forEach(this.scalarValuedFunctions::add); } public boolean isSplitLongRecords() { @@ -1435,7 +1435,7 @@ public RecordMetaData build(boolean validate) { Map> recordTypeKeyToSyntheticRecordTypeMap = Maps.newHashMapWithExpectedSize(syntheticRecordTypes.size()); RecordMetaData metaData = new RecordMetaData(recordsDescriptor, getUnionDescriptor(), unionFields, builtRecordTypes, builtSyntheticRecordTypes, recordTypeKeyToSyntheticRecordTypeMap, - indexes, universalIndexes, formerIndexes, udfs, + indexes, universalIndexes, formerIndexes, scalarValuedFunctions, splitLongRecords, storeRecordVersions, version, subspaceKeyCounter, usesSubspaceKeyCounter, recordCountKey, localFileDescriptor != null); for (RecordTypeBuilder recordTypeBuilder : recordTypes.values()) { KeyExpression primaryKey = recordTypeBuilder.getPrimaryKey(); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/ScalarValuedFunction.java similarity index 79% rename from fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java rename to fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/ScalarValuedFunction.java index 01644fbee4..924cfbc002 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Udf.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/ScalarValuedFunction.java @@ -29,13 +29,13 @@ import javax.annotation.Nonnull; /** - * Defines a User-defined-function. + * Defines a scalar User-defined-function. */ -public class Udf { +public class ScalarValuedFunction { @Nonnull private final MacroFunction macroFunction; - public Udf(@Nonnull MacroFunction functionValue) { + public ScalarValuedFunction(@Nonnull MacroFunction functionValue) { this.macroFunction = functionValue; } @@ -50,18 +50,18 @@ public String getFunctionName() { } @Nonnull - public RecordMetaDataProto.Udf toProto() { + public RecordMetaDataProto.ScalarValuedFunction toProto() { PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, PlanHashable.CURRENT_FOR_CONTINUATION); - return RecordMetaDataProto.Udf.newBuilder() + return RecordMetaDataProto.ScalarValuedFunction.newBuilder() .setFunctionValue(macroFunction.toProto(serializationContext)) .build(); } @Nonnull - public static Udf fromProto(RecordMetaDataProto.Udf proto) { + public static ScalarValuedFunction fromProto(RecordMetaDataProto.ScalarValuedFunction proto) { PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, PlanHashable.CURRENT_FOR_CONTINUATION); - return new Udf(MacroFunction.fromProto(serializationContext, proto.getFunctionValue())); + return new ScalarValuedFunction(MacroFunction.fromProto(serializationContext, proto.getFunctionValue())); } } diff --git a/fdb-record-layer-core/src/main/proto/record_metadata.proto b/fdb-record-layer-core/src/main/proto/record_metadata.proto index d2eb23b1ff..d939b0dc2d 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata.proto @@ -99,11 +99,11 @@ message MetaData { optional bool uses_subspace_key_counter = 11; repeated JoinedRecordType joined_record_types = 12; repeated UnnestedRecordType unnested_record_types = 13; - repeated Udf udfs = 14; + repeated ScalarValuedFunction scalar_valued_function = 14; extensions 1000 to 2000; } -message Udf { +message ScalarValuedFunction { optional com.apple.foundationdb.record.planprotos.PMacroFunctionValue function_value = 1; } From 1cc89a69b26a6a31bbe9988afe3437edcc655ade Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Tue, 14 Jan 2025 15:02:48 -0800 Subject: [PATCH 31/41] save --- fdb-relational-core/src/main/antlr/RelationalLexer.g4 | 1 - fdb-relational-core/src/main/antlr/RelationalParser.g4 | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 index 682011f51f..72879bb762 100644 --- a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 @@ -527,7 +527,6 @@ KEY_BLOCK_SIZE: 'KEY_BLOCK_SIZE'; LANGUAGE: 'LANGUAGE'; LAST: 'LAST'; LEAVES: 'LEAVES'; -LEFTARG: 'LEFTARG'; LESS: 'LESS'; LEVEL: 'LEVEL'; LIST: 'LIST'; diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index 62649d60b2..26b3642df5 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -797,7 +797,7 @@ ifNotExists functionCall : aggregateWindowedFunction #aggregateFunctionCall // done (supported) | specificFunction #specificFunctionCall // - | scalarFunctionName '(' functionArgs? ')' #scalarFunctionCall // done (unsupported) + | scalarFunctionName '(' functionArgs? ')' #scalarFunctionCall // done (unsupported) | userDefinedFunctionName '(' functionArgs? ')' #userDefinedFunctionCall ; From 91e6117a03362f8d098234b560e47eecbe99c976 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Tue, 14 Jan 2025 15:38:29 -0800 Subject: [PATCH 32/41] change package --- .../foundationdb/record/metadata/Key.java | 2 +- .../expressions/DimensionsKeyExpression.java | 2 +- .../expressions/EmptyKeyExpression.java | 2 +- .../expressions/FieldKeyExpression.java | 2 +- .../expressions/FunctionKeyExpression.java | 2 +- .../expressions/GroupingKeyExpression.java | 2 +- .../metadata/expressions/KeyExpression.java | 2 +- .../expressions/KeyWithValueExpression.java | 2 +- .../expressions/ListKeyExpression.java | 2 +- .../expressions/LiteralKeyExpression.java | 2 +- .../expressions/NestingKeyExpression.java | 2 +- .../expressions/RecordTypeKeyExpression.java | 2 +- .../expressions/SplitKeyExpression.java | 2 +- .../expressions/ThenKeyExpression.java | 2 +- .../expressions/VersionKeyExpression.java | 2 +- .../KeyExpressionExpansionVisitor.java | 2 +- .../plan/cascades/NullableArrayTypeUtils.java | 2 +- .../main/proto/record_key_expression.proto | 2 +- .../src/main/proto/record_metadata.proto | 24 +++++++++---------- .../main/proto/record_metadata_options.proto | 2 +- .../src/main/proto/record_query_plan.proto | 20 ++++++++-------- .../src/main/proto/record_query_runtime.proto | 1 - .../main/proto/lucene_record_query_plan.proto | 2 +- 23 files changed, 42 insertions(+), 43 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java index c930bdd688..1e38ba7be4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java @@ -22,7 +22,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression; import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java index 30a74bcbe3..3de25ac5a1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java index 4068d36946..fd7dc20c91 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java index 6cf17f350c..aba60e09db 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java @@ -24,7 +24,7 @@ import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java index 6194200f40..0d2884e628 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FunctionKeyExpression.java @@ -25,7 +25,7 @@ import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.RecordCoreArgumentException; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.logging.LogMessageKeys; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java index 916cf3fd0f..bb2a3f0c60 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/GroupingKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java index 06caf442de..7d660c03fe 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyExpression.java @@ -24,7 +24,7 @@ import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.QueryHashable; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.ExpansionVisitor; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java index 37bd7acda4..db7ce74510 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/KeyWithValueExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java index 0a71db1f51..fcd77a95fb 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ListKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java index 01e661c067..b8bf1a088d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpression.java @@ -24,7 +24,7 @@ import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java index 321c3fa133..c9a2882b00 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/NestingKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java index abfc849f6e..461cbb39da 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/RecordTypeKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java index d015c84457..0f7aba1519 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/SplitKeyExpression.java @@ -24,7 +24,7 @@ import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java index 69fc395d9c..b520650607 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/ThenKeyExpression.java @@ -24,7 +24,7 @@ import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java index 3859a78f6c..89d4f846f7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/VersionKeyExpression.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.ObjectPlanHash; import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordVersion; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java index 8cb1da0112..d0e034f6b2 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/KeyExpressionExpansionVisitor.java @@ -21,7 +21,7 @@ package com.apple.foundationdb.record.query.plan.cascades; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression; import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; import com.apple.foundationdb.record.metadata.expressions.FunctionKeyExpression; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/NullableArrayTypeUtils.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/NullableArrayTypeUtils.java index f741723aea..4686a68d06 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/NullableArrayTypeUtils.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/NullableArrayTypeUtils.java @@ -20,7 +20,7 @@ package com.apple.foundationdb.record.query.plan.cascades; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.expressions.NestingKeyExpression; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.MessageHelpers; diff --git a/fdb-record-layer-core/src/main/proto/record_key_expression.proto b/fdb-record-layer-core/src/main/proto/record_key_expression.proto index 9c390e14b3..b351ea9367 100644 --- a/fdb-record-layer-core/src/main/proto/record_key_expression.proto +++ b/fdb-record-layer-core/src/main/proto/record_key_expression.proto @@ -19,7 +19,7 @@ */ syntax = "proto2"; -package com.apple.foundationdb.record; +package com.apple.foundationdb.record.expressions; import "google/protobuf/descriptor.proto"; option java_outer_classname = "RecordKeyExpressionProto"; diff --git a/fdb-record-layer-core/src/main/proto/record_metadata.proto b/fdb-record-layer-core/src/main/proto/record_metadata.proto index d939b0dc2d..5fa78cc3f8 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata.proto @@ -30,7 +30,7 @@ message DataStoreInfo { optional int32 formatVersion = 1; optional int32 metaDataversion = 2; optional int32 userVersion = 3; - optional KeyExpression record_count_key = 4; + optional com.apple.foundationdb.record.expressions.KeyExpression record_count_key = 4; optional uint64 lastUpdateTime = 5; optional bool omit_unsplit_record_suffix = 6; optional bool cacheable = 7; @@ -53,11 +53,11 @@ message Index { repeated string record_type = 1; optional Type index_type = 2 [deprecated = true]; // Use type. optional string name = 3; - optional KeyExpression root_expression = 4; + optional com.apple.foundationdb.record.expressions.KeyExpression root_expression = 4; optional bytes subspace_key = 5; optional int32 last_modified_version = 6; // value_expression should now be expressed as a KeyWithValue() root expression - optional KeyExpression value_expression = 7 [deprecated = true]; + optional com.apple.foundationdb.record.expressions.KeyExpression value_expression = 7 [deprecated = true]; optional string type = 8; repeated Option options = 9; message Option { @@ -71,9 +71,9 @@ message Index { message RecordType { required string name = 1; - optional KeyExpression primary_key = 2; + optional com.apple.foundationdb.record.expressions.KeyExpression primary_key = 2; optional int32 since_version = 3; - optional Value explicit_key = 4; + optional com.apple.foundationdb.record.expressions.Value explicit_key = 4; extensions 1000 to 2000; } @@ -92,7 +92,7 @@ message MetaData { optional bool split_long_records = 4; optional int32 version = 5; repeated FormerIndex former_indexes = 6; - optional KeyExpression record_count_key = 7 [deprecated = true]; + optional com.apple.foundationdb.record.expressions.KeyExpression record_count_key = 7 [deprecated = true]; optional bool store_record_versions = 8; repeated google.protobuf.FileDescriptorProto dependencies = 9; optional int64 subspace_key_counter = 10; @@ -109,7 +109,7 @@ message ScalarValuedFunction { message JoinedRecordType { optional string name = 1; - optional Value record_type_key = 4; + optional com.apple.foundationdb.record.expressions.Value record_type_key = 4; message JoinConstituent { optional string name = 1; @@ -119,9 +119,9 @@ message JoinedRecordType { message Join { optional string left = 1; - optional KeyExpression left_expression = 2; + optional com.apple.foundationdb.record.expressions.KeyExpression left_expression = 2; optional string right = 3; - optional KeyExpression right_expression = 4; + optional com.apple.foundationdb.record.expressions.KeyExpression right_expression = 4; } repeated JoinConstituent join_constituents = 10; @@ -130,13 +130,13 @@ message JoinedRecordType { message UnnestedRecordType { optional string name = 1; - optional Value record_type_key = 2; + optional com.apple.foundationdb.record.expressions.Value record_type_key = 2; message NestedConstituent { optional string name = 1; optional string parent = 2; optional string type_name = 3; - optional KeyExpression nesting_expression = 4; + optional com.apple.foundationdb.record.expressions.KeyExpression nesting_expression = 4; } repeated NestedConstituent nested_constituents = 3; @@ -158,7 +158,7 @@ enum ComparisonType { message SimpleComparison { optional ComparisonType type = 1; - optional Value operand = 2; + optional com.apple.foundationdb.record.expressions.Value operand = 2; } // in Comparisons.Comparison we have a special NullComparison that takes care of comparing diff --git a/fdb-record-layer-core/src/main/proto/record_metadata_options.proto b/fdb-record-layer-core/src/main/proto/record_metadata_options.proto index c4426b9ef1..26ecbeb261 100644 --- a/fdb-record-layer-core/src/main/proto/record_metadata_options.proto +++ b/fdb-record-layer-core/src/main/proto/record_metadata_options.proto @@ -44,7 +44,7 @@ message RecordTypeOptions { } optional Usage usage = 1 [default = UNSET]; optional int32 since_version = 2; // This record_type is introduced on this metadata version - optional Value record_type_key = 3; + optional com.apple.foundationdb.record.expressions.Value record_type_key = 3; } extend google.protobuf.MessageOptions { diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 9370202a12..16673652f2 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -975,7 +975,7 @@ message PSimpleComparison { message PConversionSimpleComparison { optional PComparison.PComparisonType type = 1; optional PComparableObject object = 2; - optional KeyExpression conversion = 3; + optional com.apple.foundationdb.record.expressions.KeyExpression conversion = 3; } message PParameterComparison { @@ -995,7 +995,7 @@ message PConversionParameterComparison { optional PComparison.PComparisonType type = 1; optional string parameter = 2; optional PParameterComparison.PBindingKind internal = 3; - optional KeyExpression conversion = 4; + optional com.apple.foundationdb.record.expressions.KeyExpression conversion = 4; } message PValueComparison { @@ -1021,7 +1021,7 @@ message PMultiColumnComparison { } message PInvertedFunctionComparison { - optional Function function = 1; + optional com.apple.foundationdb.record.expressions.Function function = 1; optional PComparison original_comparison = 2; optional PComparison.PComparisonType type = 3; } @@ -1346,7 +1346,7 @@ message PRecordQueryFlatMapPlan { // message PRecordQueryIndexPlan { optional string index_name = 1; - optional KeyExpression common_primary_key = 2; + optional com.apple.foundationdb.record.expressions.KeyExpression common_primary_key = 2; optional PIndexScanParameters scan_parameters = 3; optional PIndexFetchMethod index_fetch_method = 4; optional PFetchIndexRecords fetch_index_records = 5; @@ -1515,7 +1515,7 @@ message PRecordQueryIntersectionPlan { message PComparisonKeyFunction { message POnKeyExpression { - optional KeyExpression comparison_key_expression = 1; + optional com.apple.foundationdb.record.expressions.KeyExpression comparison_key_expression = 1; } message POnValues { optional string base_alias = 1; @@ -1606,7 +1606,7 @@ message PRecordQuerySortPlan { } message PRecordQuerySortKey { - optional KeyExpression key = 1; + optional com.apple.foundationdb.record.expressions.KeyExpression key = 1; optional bool reverse = 2; } @@ -1617,7 +1617,7 @@ message PRecordQueryScanPlan { optional bool has_record_types = 1; repeated string record_types = 2; optional PType flowed_type = 3; - optional KeyExpression common_primary_key = 4; + optional com.apple.foundationdb.record.expressions.KeyExpression common_primary_key = 4; optional PScanComparisons comparisons = 5; optional bool reverse = 6; optional bool strictly_sorted = 7; @@ -1651,7 +1651,7 @@ message PScoreForRank { message PIndexAggregateFunction { optional string name = 1; - optional KeyExpression operand = 2; + optional com.apple.foundationdb.record.expressions.KeyExpression operand = 2; optional string index = 3; } @@ -1704,7 +1704,7 @@ message PRecordQueryUnionPlan { // message PRecordQueryUnorderedDistinctPlan { optional PPhysicalQuantifier inner = 1; - optional KeyExpression comparison_key = 2; + optional com.apple.foundationdb.record.expressions.KeyExpression comparison_key = 2; } // @@ -1752,7 +1752,7 @@ message PFDBRecordVersion { message PComparableObject { oneof specific_object { - Value primitive_object = 1; + com.apple.foundationdb.record.expressions.Value primitive_object = 1; PEnumLightValue enum_object = 2; PUUID uuid = 3; PFDBRecordVersion fdb_record_version = 4; diff --git a/fdb-record-layer-core/src/main/proto/record_query_runtime.proto b/fdb-record-layer-core/src/main/proto/record_query_runtime.proto index 73491c934b..0a8d67bca8 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_runtime.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_runtime.proto @@ -21,7 +21,6 @@ syntax = "proto2"; package com.apple.foundationdb.record.planprotos; import "google/protobuf/descriptor.proto"; -import "record_metadata.proto"; import "record_query_plan.proto"; option java_outer_classname = "RecordQueryRuntimeProto"; diff --git a/fdb-record-layer-lucene/src/main/proto/lucene_record_query_plan.proto b/fdb-record-layer-lucene/src/main/proto/lucene_record_query_plan.proto index 76a1e307aa..760f2b012e 100644 --- a/fdb-record-layer-lucene/src/main/proto/lucene_record_query_plan.proto +++ b/fdb-record-layer-lucene/src/main/proto/lucene_record_query_plan.proto @@ -30,7 +30,7 @@ option java_multiple_files = true; message PLuceneIndexQueryPlan { optional PRecordQueryIndexPlan super = 1; optional bool has_stored_fields = 2; - repeated KeyExpression stored_fields = 3; + repeated com.apple.foundationdb.record.expressions.KeyExpression stored_fields = 3; } message PLuceneScanParameters { From 262f03bb70a4572ea96fde484cacd061705aff57 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Tue, 14 Jan 2025 16:36:46 -0800 Subject: [PATCH 33/41] fix test compile error --- .../record/metadata/MetaDataEvolutionValidatorTest.java | 2 +- .../apple/foundationdb/record/metadata/MetaDataProtoTest.java | 2 +- .../foundationdb/record/metadata/UnknownKeyExpression.java | 2 +- .../record/metadata/expressions/LiteralKeyExpressionTest.java | 2 +- .../record/provider/foundationdb/FDBMetaDataStoreTest.java | 2 +- .../record/provider/foundationdb/FDBRecordStoreTest.java | 2 +- .../foundationdb/storestate/FDBRecordStoreStateCacheTest.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidatorTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidatorTest.java index 79f98a7d70..c888cd0027 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidatorTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataEvolutionValidatorTest.java @@ -22,7 +22,7 @@ import com.apple.foundationdb.async.RankedSet; import com.apple.foundationdb.record.RecordCoreException; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataBuilder; import com.apple.foundationdb.record.RecordMetaDataOptionsProto; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataProtoTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataProtoTest.java index 67f925abf1..34c64f23e3 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataProtoTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/MetaDataProtoTest.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataBuilder; import com.apple.foundationdb.record.RecordMetaDataOptionsProto; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.TestRecords1Proto; import com.apple.foundationdb.record.TestRecords2Proto; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java index 04a884ad9e..0c6b5be7a9 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/UnknownKeyExpression.java @@ -21,7 +21,7 @@ package com.apple.foundationdb.record.metadata; import com.apple.foundationdb.record.ObjectPlanHash; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.expressions.BaseKeyExpression; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.provider.foundationdb.FDBRecord; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java index 86e8815fb0..98957d74c6 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/metadata/expressions/LiteralKeyExpressionTest.java @@ -20,7 +20,7 @@ package com.apple.foundationdb.record.metadata.expressions; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.provider.common.text.TextSamples; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java index b60f851094..c0cbe34eca 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBMetaDataStoreTest.java @@ -26,7 +26,7 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataBuilder; import com.apple.foundationdb.record.RecordMetaDataOptionsProto; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.TestNoUnionEvolvedIllegalProto; import com.apple.foundationdb.record.TestNoUnionEvolvedProto; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTest.java index 128fe3dbdc..a9f3a15bd6 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/FDBRecordStoreTest.java @@ -35,7 +35,7 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataBuilder; import com.apple.foundationdb.record.RecordMetaDataOptionsProto; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.RecordMetaDataProvider; import com.apple.foundationdb.record.ScanProperties; diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/storestate/FDBRecordStoreStateCacheTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/storestate/FDBRecordStoreStateCacheTest.java index 17a5a50429..5eec68fc94 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/storestate/FDBRecordStoreStateCacheTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/provider/foundationdb/storestate/FDBRecordStoreStateCacheTest.java @@ -25,7 +25,7 @@ import com.apple.foundationdb.record.RecordCoreException; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataBuilder; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.TestRecords1Proto; import com.apple.foundationdb.record.metadata.Key; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; From e8c4dfc9e21c025b9fb9198eb86aa629a15a4d65 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Tue, 14 Jan 2025 23:26:14 -0800 Subject: [PATCH 34/41] change --- .../recordlayer/CatalogMetaData.java | 2 +- .../metadata/RecordLayerSchemaTemplate.java | 42 ++++++------ .../metadata/RecordLayerTable.java | 24 +------ .../RecordLayerUserDefinedFunction.java | 64 ------------------- .../serde/RecordMetadataDeserializer.java | 4 +- .../serde/RecordMetadataSerializer.java | 2 +- .../recordlayer/query/SemanticAnalyzer.java | 8 ++- .../recordlayer/query/TautologicalValue.java | 6 -- .../query/functions/SqlFunctionCatalog.java | 17 +++-- .../query/visitors/BaseVisitor.java | 4 +- .../query/visitors/DdlVisitor.java | 8 +-- .../query/visitors/DelegatingVisitor.java | 2 +- .../query/visitors/ExpressionVisitor.java | 7 +- .../query/visitors/TypedVisitor.java | 4 ++ .../relational/util/NullableArrayUtils.java | 2 +- .../relational/api/DriverManagerTest.java | 2 +- .../api/ddl/DdlStatementParsingTest.java | 2 +- .../query/NullableArrayUtilsTest.java | 3 +- .../recordlayer/query/StandardQueryTests.java | 2 +- 19 files changed, 57 insertions(+), 148 deletions(-) delete mode 100644 fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerUserDefinedFunction.java diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java index ae880f0cd6..2a4a901315 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java @@ -23,7 +23,7 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.RecordMetaData; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.MetaDataException; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java index eab8884a8e..1caea53bf2 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java @@ -25,13 +25,12 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.Key; -import com.apple.foundationdb.record.metadata.Udf; +import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.query.combinatorics.TopologicalSort; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.api.metadata.Index; -import com.apple.foundationdb.relational.api.metadata.FunctionDefinition; import com.apple.foundationdb.relational.api.metadata.SchemaTemplate; import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.api.metadata.Visitor; @@ -72,7 +71,8 @@ public final class RecordLayerSchemaTemplate implements SchemaTemplate { private final Set tables; @Nonnull - private final Set udfs; + private final Set scalarValuedFunctions; + private final int version; private final boolean enableLongRows; @@ -90,12 +90,13 @@ public final class RecordLayerSchemaTemplate implements SchemaTemplate { private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, - @Nonnull final Set udfs, + @Nonnull final Set scalarValuedFunctions, int version, boolean enableLongRows, boolean storeRowVersions) { this.name = name; this.tables = tables; + this.scalarValuedFunctions = scalarValuedFunctions; this.version = version; this.enableLongRows = enableLongRows; this.storeRowVersions = storeRowVersions; @@ -106,7 +107,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, - @Nonnull final Set udfs, + @Nonnull final Set scalarValuedFunctions, int version, boolean enableLongRows, boolean storeRowVersions, @@ -114,6 +115,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, this.name = name; this.version = version; this.tables = tables; + this.scalarValuedFunctions = scalarValuedFunctions; this.enableLongRows = enableLongRows; this.storeRowVersions = storeRowVersions; this.metaDataSupplier = Suppliers.memoize(() -> cachedMetadata); @@ -155,8 +157,8 @@ public RecordLayerSchema generateSchema(@Nonnull String databaseId, @Nonnull Str } @Nonnull - public Set getAllUdfs() { - return udfs; + public Set getAllScalarValuedFunctions() { + return scalarValuedFunctions; } @Nonnull @@ -190,7 +192,6 @@ public RecordMetaData toRecordMetadata() { public static RecordLayerSchemaTemplate fromRecordMetadata(@Nonnull RecordMetaData metaData, @Nonnull String templateName, int version) { - System.out.println("RecordLayerSchemaTemplate.fromRecordMetadata is called"); final var deserializer = new RecordMetadataDeserializer(metaData); final var builder = deserializer.getSchemaTemplate(templateName, version); return builder.setCachedMetadata(metaData).build(); @@ -211,7 +212,6 @@ public static RecordLayerSchemaTemplate fromRecordMetadataWithFakeTemplateNameAn @Override public Optional
findTableByName(@Nonnull final String tableName) { for (final var table : getTables()) { - System.out.println("findTableByName:" + table.getName()); if (table.getName().equals(tableName)) { return Optional.of(table); } @@ -323,12 +323,15 @@ public static final class Builder { private final Map tables; + private final Set scalarValuedFunctions; + private final Map auxiliaryTypes; // for quick lookup private RecordMetaData cachedMetadata; private Builder() { tables = new LinkedHashMap<>(); + scalarValuedFunctions = new HashSet<>(); auxiliaryTypes = new LinkedHashMap<>(); // enable long rows is TRUE by default enableLongRows = true; @@ -417,15 +420,14 @@ public Builder addAuxiliaryTypes(@Nonnull Collection auxiliaryTy } @Nonnull - public Builder addUdf(@Nonnull Udf Udf) { - Assert.thatUnchecked(!udfMap.containsKey(Udf.getFunctionName()), ErrorCode.INVALID_SCHEMA_TEMPLATE, TABLE_ALREADY_EXISTS, Udf.getFunctionName()); - udfMap.put(Udf.getFunctionName(), Udf); + public Builder addScalarValuedFunction(@Nonnull ScalarValuedFunction scalarValuedFunction) { + scalarValuedFunctions.add(scalarValuedFunction); return this; } @Nonnull - public Builder addUdfs(@Nonnull Collection Udf) { - Udf.forEach(this::addUdf); + public Builder addScalarValuedFunctions(@Nonnull Collection scalarValuedFunctions) { + scalarValuedFunctions.forEach(this::addScalarValuedFunction); return this; } @@ -447,13 +449,6 @@ public RecordLayerTable extractTable(@Nonnull final String name) { return tables.remove(name); } - @Nonnull - public DataType.Named extractAuxiliaryType(@Nonnull final String name) { - System.out.println("extractAuxiliaryType called with:" + name + " map:" + auxiliaryTypes); - Assert.thatUnchecked(auxiliaryTypes.containsKey(name.toUpperCase(Locale.ROOT)), ErrorCode.UNDEFINED_TABLE, "could not find '%s'", name); - return auxiliaryTypes.remove(name.toUpperCase(Locale.ROOT)); - } - @Nonnull public Optional findType(@Nonnull final String name) { // we should also check whether the name exists in _both_ databases. @@ -493,11 +488,10 @@ public RecordLayerSchemaTemplate build() { if (needsResolution) { resolveTypes(); } - System.out.println("RecordLayerSchemaTemplate name:" + name + " is build with auxiliary type size:" + auxiliaryTypes.size()); if (cachedMetadata != null) { - return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), version, enableLongRows, storeRowVersions, cachedMetadata); + return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), scalarValuedFunctions, version, enableLongRows, storeRowVersions, cachedMetadata); } else { - return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), version, enableLongRows, storeRowVersions); + return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), scalarValuedFunctions, version, enableLongRows, storeRowVersions); } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java index 805257788a..12ce0cf0d3 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java @@ -63,9 +63,6 @@ public final class RecordLayerTable implements Table { @Nonnull private final Set indexes; - @Nonnull - private final Set operatorDefinitions; - @Nonnull private final KeyExpression primaryKey; @@ -81,7 +78,6 @@ public final class RecordLayerTable implements Table { private RecordLayerTable(@Nonnull final String name, @Nonnull final List columns, @Nonnull final Set indexes, - @Nonnull final Set operatorDefinitions, @Nonnull final KeyExpression primaryKey, @Nonnull final Map generations, final DataType.StructType dataType, @@ -89,7 +85,6 @@ private RecordLayerTable(@Nonnull final String name, this.name = name; this.columns = ImmutableList.copyOf(columns); this.indexes = ImmutableSet.copyOf(indexes); - this.operatorDefinitions = ImmutableSet.copyOf(operatorDefinitions); this.primaryKey = primaryKey; this.generations = generations; this.dataType = dataType == null ? calculateDataType() : dataType; @@ -108,11 +103,6 @@ public Set getIndexes() { return indexes; } - @Nonnull - public Set getOperatorDefinitions() { - return operatorDefinitions; - } - @Nonnull public Map getGenerations() { return generations; @@ -215,9 +205,6 @@ public static final class Builder { @Nonnull private Set indexes; - @Nonnull - private Set operatorDefinitions; - @Nonnull private ImmutableList.Builder columns; @@ -271,13 +258,6 @@ public Builder addIndex(@Nonnull final RecordLayerIndex index) { return this; } - @Nonnull - public Builder addOperatorDefinition(@Nonnull final RecordLayerUserDefinedFunction userDefinedOperator) { - Assert.thatUnchecked(operatorDefinitions.stream().noneMatch(i -> userDefinedOperator.getName().equals(i.getName())), ErrorCode.INDEX_ALREADY_EXISTS, "attempt to add duplicate index '%s'", userDefinedOperator.getName()); - this.operatorDefinitions.add(userDefinedOperator); - return this; - } - @Nonnull public Builder addGenerations(@Nonnull final Map generations) { generations.forEach(this::addGeneration); @@ -357,9 +337,7 @@ public RecordLayerTable build() { Assert.thatUnchecked(!columnsList.isEmpty(), ErrorCode.INVALID_TABLE_DEFINITION, "Attempting to create table %s without columns", name); final var indexesSet = ImmutableSet.copyOf(indexes); - final Set operatorDefinitionsSet = operatorDefinitions == null ? ImmutableSet.of() : ImmutableSet.copyOf(operatorDefinitions); - - return new RecordLayerTable(name, columnsList, indexesSet, operatorDefinitionsSet, getPrimaryKey(), generations, dataType, record); + return new RecordLayerTable(name, columnsList, indexesSet, getPrimaryKey(), generations, dataType, record); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerUserDefinedFunction.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerUserDefinedFunction.java deleted file mode 100644 index 0dcce36cc1..0000000000 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerUserDefinedFunction.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * RecordLayerUserDefinedFunction.java - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2021-2024 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apple.foundationdb.relational.recordlayer.metadata; - -import com.apple.foundationdb.relational.api.metadata.FunctionDefinition; -import com.apple.foundationdb.relational.api.metadata.Visitor; -import com.apple.foundationdb.relational.generated.RelationalParser; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.RuleContext; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.TerminalNode; - -import javax.annotation.Nonnull; -import java.util.List; - -public class RecordLayerUserDefinedFunction implements FunctionDefinition { - @Nonnull private final String functionName; - @Nonnull private final String inputTypeName; - @Nonnull private final List returnColumnTypes; - @Nonnull private final String param; - @Nonnull private final List functionCalls; - - public RecordLayerUserDefinedFunction(@Nonnull String functionName, @Nonnull String inputTypeName, @Nonnull List returnColumnTypes, @Nonnull String param, @Nonnull List functionCalls) { - this.functionName = functionName; - this.inputTypeName = inputTypeName; - this.returnColumnTypes = returnColumnTypes; - this.param = param; - this.functionCalls = functionCalls; - } - - @Nonnull - @Override - public String getName() { - return functionName; - } - - @Override - public void accept(@Nonnull Visitor visitor) { - visitor.visit(this); - } - - @Nonnull - public String getInputTypeName() { - return inputTypeName; - } -} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java index b7e8403957..78b2c81ae8 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java @@ -60,8 +60,8 @@ public RecordLayerSchemaTemplate.Builder getSchemaTemplate(@Nonnull final String .setEnableLongRows(recordMetaData.isSplitLongRecords()) .setName(schemaTemplateName) .setIntermingleTables(!recordMetaData.primaryKeyHasRecordTypePrefix()); - for (final var u: recordMetaData.getAllUdfs()) { - schemaTemplateBuilder.addUdf(u); + for (final var u: recordMetaData.getAllScalarValuedFunctions()) { + schemaTemplateBuilder.addScalarValuedFunction(u); } final var nameToTableBuilder = new HashMap(); for (final var registeredType : registeredTypes) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java index 830a1ce342..94690eabe4 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java @@ -87,7 +87,7 @@ public void visit(@Nonnull com.apple.foundationdb.relational.api.metadata.Index @Override public void visit(@Nonnull SchemaTemplate schemaTemplate) { Assert.thatUnchecked(schemaTemplate instanceof RecordLayerSchemaTemplate); - getBuilder().addUdfs(((RecordLayerSchemaTemplate) schemaTemplate).getAllUdfs()); + getBuilder().addScalarValuedFunctions(((RecordLayerSchemaTemplate) schemaTemplate).getAllScalarValuedFunctions()); getBuilder().setSplitLongRecords(schemaTemplate.isEnableLongRows()); getBuilder().setStoreRecordVersions(schemaTemplate.isStoreRowVersions()); getBuilder().setVersion(schemaTemplate.getVersion()); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index 57d429d07e..973ea46de1 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -21,7 +21,7 @@ package com.apple.foundationdb.relational.recordlayer.query; import com.apple.foundationdb.annotation.API; -import com.apple.foundationdb.record.metadata.Udf; +import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.query.plan.cascades.AccessHint; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; import com.apple.foundationdb.record.query.plan.cascades.Correlated; @@ -31,6 +31,7 @@ import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; import com.apple.foundationdb.record.query.plan.cascades.values.*; +import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.api.metadata.DataType; @@ -84,8 +85,8 @@ public SemanticAnalyzer(@Nonnull SchemaTemplate metadataCatalog, this.metadataCatalog = metadataCatalog; this.functionCatalog = functionCatalog; // add UDFs to functionCatalog - for (Udf udf: ((RecordLayerSchemaTemplate) metadataCatalog).getAllUdfs()) { - ((SqlFunctionCatalog)this.functionCatalog).addFunction(udf); + for (ScalarValuedFunction scalarValuedFunction: ((RecordLayerSchemaTemplate) metadataCatalog).getAllScalarValuedFunctions()) { + ((SqlFunctionCatalog)this.functionCatalog).addFunction(scalarValuedFunction); } } @@ -765,6 +766,7 @@ public static void validateContinuation(@Nonnull Expression continuation) { @Nonnull public Expression resolveFunction(@Nonnull String functionName, boolean flattenSingleItemRecords, @Nonnull Expression... arguments) { + System.out.println("functionName:" + functionName); Assert.thatUnchecked(functionCatalog.containsFunction(functionName), ErrorCode.UNSUPPORTED_QUERY, () -> String.format("Unsupported operator %s", functionName)); final var builtInFunction = functionCatalog.lookUpFunction(functionName); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/TautologicalValue.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/TautologicalValue.java index 56b250adb5..cde352e254 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/TautologicalValue.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/TautologicalValue.java @@ -82,12 +82,6 @@ protected Iterable computeChildren() { return ImmutableList.of(); } - @Nonnull - @Override - public ExplainTokensWithPrecedence explain(@Nonnull Iterable> iterable) { - return null; - } - @Nullable @Override public Object eval(@Nonnull FDBRecordStoreBase store, @Nonnull EvaluationContext context) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java index 6df27090b9..2ee090eac0 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java @@ -21,7 +21,7 @@ package com.apple.foundationdb.relational.recordlayer.query.functions; import com.apple.foundationdb.annotation.API; -import com.apple.foundationdb.record.metadata.Udf; +import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; import com.apple.foundationdb.record.query.plan.cascades.Function; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; @@ -32,7 +32,10 @@ import com.google.common.collect.ImmutableMap; import javax.annotation.Nonnull; +import java.util.HashMap; import java.util.Locale; +import java.util.Map; +import java.util.Objects; import java.util.stream.StreamSupport; import static java.util.stream.Collectors.toList; @@ -53,7 +56,7 @@ public final class SqlFunctionCatalog implements FunctionCatalog { private final ImmutableMap> synonyms; @Nonnull - private final Map> udfMap; + private final Map> scalarValuedFunctionMap = new HashMap<>(); private SqlFunctionCatalog() { this.synonyms = createSynonyms(); @@ -65,12 +68,12 @@ public Function lookUpFunction(@Nonnull String name) { if (synonyms.get(name.toLowerCase(Locale.ROOT)) != null) { return Objects.requireNonNull(synonyms.get(name.toLowerCase(Locale.ROOT))); } - return Assert.notNullUnchecked(udfMap.get(name.toLowerCase(Locale.ROOT))); + return Assert.notNullUnchecked(scalarValuedFunctionMap.get(name.toLowerCase(Locale.ROOT))); } @Override public boolean containsFunction(@Nonnull String name) { - return synonyms.containsKey(name.toLowerCase(Locale.ROOT)); + return synonyms.containsKey(name.toLowerCase(Locale.ROOT)) || scalarValuedFunctionMap.containsKey(name.toLowerCase(Locale.ROOT)); } @Nonnull @@ -123,9 +126,9 @@ public static SqlFunctionCatalog instance() { } @Nonnull - public Function addFunction(@Nonnull Udf udf) { - final var func = udf.getMacroFunction(); - udfMap.put(udf.getFunctionName(), func); + public Function addFunction(@Nonnull ScalarValuedFunction scalarValuedFunction) { + final var func = scalarValuedFunction.getMacroFunction(); + scalarValuedFunctionMap.put(scalarValuedFunction.getFunctionName(), func); return func; } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index 36d7fde3fc..44bef4325c 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -399,8 +399,8 @@ public Object visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefiniti @Nonnull @Override - public Object visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { - return visitChildren(ctx); + public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { + return expressionVisitor.visitUserDefinedFunctionCall(ctx); } @Override diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 6a8d69c658..0e462d6614 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -21,7 +21,7 @@ package com.apple.foundationdb.relational.recordlayer.query.visitors; import com.apple.foundationdb.annotation.API; -import com.apple.foundationdb.record.metadata.Udf; +import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.MacroFunction; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalSortExpression; @@ -180,7 +180,7 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi @Nonnull @Override - public Udf visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { + public ScalarValuedFunction visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { final var ddlCatalog = metadataBuilder.build(); // parse the function definition using the newly constructed metadata. getDelegate().replaceCatalog(ddlCatalog); @@ -201,7 +201,7 @@ public Udf visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionC Optional fieldValue = semanticAnalyzer.lookUpNestedField(functionBody, paramNameId, argumentValue, returnType); Assert.thatUnchecked(fieldValue.isPresent(), "couldn't resolve function definition"); - return new Udf(new MacroFunction(ctx.functionName.getText(), List.of(argumentValue.getResultType()), List.of(argumentValue.getAlias()), fieldValue.get())); + return new ScalarValuedFunction(new MacroFunction(ctx.functionName.getText(), List.of(argumentValue), fieldValue.get())); } @Nonnull @@ -264,7 +264,7 @@ public ProceduralPlan visitCreateSchemaTemplateStatement(@Nonnull RelationalPars metadataBuilder.addTable(tableWithIndex); } final var udfs = functionClauses.build().stream().map(this::visitFunctionDefinition).collect(ImmutableList.toImmutableList()); - metadataBuilder.addUdfs(udfs); + metadataBuilder.addScalarValuedFunctions(udfs); return ProceduralPlan.of(metadataOperationsFactory.getCreateSchemaTemplateConstantAction(metadataBuilder.build(), Options.NONE)); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index f2b09f4dfb..8ed47b4842 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -237,7 +237,7 @@ public Object visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefiniti @Nonnull @Override - public Object visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { + public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { return getDelegate().visitUserDefinedFunctionCall(ctx); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index f0a81c044d..6b15f0bb6d 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -21,7 +21,6 @@ package com.apple.foundationdb.relational.recordlayer.query.visitors; import com.apple.foundationdb.annotation.API; -import com.apple.foundationdb.record.metadata.Udf; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.AbstractArrayConstructorValue; @@ -36,8 +35,6 @@ import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; -import com.apple.foundationdb.relational.api.metadata.DataType; -import com.apple.foundationdb.relational.api.metadata.FunctionDefinition; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; import com.apple.foundationdb.relational.recordlayer.query.Expression; @@ -118,6 +115,7 @@ public Expression visitFullColumnNameExpressionAtom(@Nonnull RelationalParser.Fu @Nonnull @Override public Expressions visitSelectElements(@Nonnull RelationalParser.SelectElementsContext selectElementsContext) { + System.out.println("visitSelectElements is called"); return Expressions.of(selectElementsContext.selectElement().stream() .map(selectElement -> Assert.castUnchecked(selectElement.accept(this), Expression.class)) .collect(ImmutableList.toImmutableList())); @@ -183,7 +181,6 @@ public Expression visitGroupByItem(@Nonnull RelationalParser.GroupByItemContext @Nonnull @Override public Expression visitAggregateFunctionCall(@Nonnull RelationalParser.AggregateFunctionCallContext functionCon) { - System.out.println("visitAggregateFunctionCall called"); return visitAggregateWindowedFunction(functionCon.aggregateWindowedFunction()); } @@ -254,6 +251,7 @@ public Expression visitCaseFunctionCall(@Nonnull RelationalParser.CaseFunctionCa @Nonnull @Override public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { + System.out.println("visitUserDefinedFunctionCall called"); final var functionName = ctx.userDefinedFunctionName().getText(); Expressions arguments = visitFunctionArgs(ctx.functionArgs()); return getDelegate().resolveFunction(functionName, arguments.asList().toArray(new Expression[0])); @@ -704,6 +702,7 @@ public Expression visitArrayConstructor(@Nonnull RelationalParser.ArrayConstruct @Nonnull private Expressions parseRecordFields(@Nonnull List parserRuleContexts, @Nullable List targetFields) { + System.out.println("parseRecordFields is called"); Assert.thatUnchecked(targetFields == null || targetFields.size() == parserRuleContexts.size()); final var resultsBuilder = ImmutableList.builder(); for (int i = 0; i < parserRuleContexts.size(); i++) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java index 546f271a90..65fc12cf56 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java @@ -706,6 +706,10 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override Expression visitScalarFunctionCall(@Nonnull RelationalParser.ScalarFunctionCallContext ctx); + @Nonnull + @Override + Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx); + @Nonnull @Override Object visitSimpleFunctionCall(@Nonnull RelationalParser.SimpleFunctionCallContext ctx); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/util/NullableArrayUtils.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/util/NullableArrayUtils.java index 213e487550..db4478cbc8 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/util/NullableArrayUtils.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/util/NullableArrayUtils.java @@ -21,7 +21,7 @@ package com.apple.foundationdb.relational.util; import com.apple.foundationdb.annotation.API; -import com.apple.foundationdb.record.RecordKeyExpressionProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.TypeRepository; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java index 5948d12ea7..82475c98e4 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java @@ -20,7 +20,6 @@ package com.apple.foundationdb.relational.api; -import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -30,6 +29,7 @@ import java.sql.DriverManager; import java.sql.SQLException; +import java.util.stream.Collectors; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java index 1084071039..1121766bc9 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/ddl/DdlStatementParsingTest.java @@ -20,10 +20,10 @@ package com.apple.foundationdb.relational.api.ddl; -import com.apple.foundationdb.record.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.RecordStoreState; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.metadata.expressions.ThenKeyExpression; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/NullableArrayUtilsTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/NullableArrayUtilsTest.java index 3de4c7150a..1c9c36f0d7 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/NullableArrayUtilsTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/NullableArrayUtilsTest.java @@ -20,8 +20,7 @@ package com.apple.foundationdb.relational.recordlayer.query; -import com.apple.foundationdb.record.RecordKeyExpressionProto; -import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.relational.util.NullableArrayUtils; import com.google.protobuf.DescriptorProtos; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index 162ba9efaa..1f933205e5 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -932,7 +932,7 @@ void testUserDefinedFunction3() throws Exception { "CREATE TYPE AS STRUCT Location (name string, coord LATLON)" + "CREATE TABLE T1(uid bigint, loc Location, PRIMARY KEY(uid))\n" + "CREATE FUNCTION lat(x Location) RETURNS string AS x.coord.latitude\n" + - "CREATE FUNCTION name(x Location) RETURNS string AS x.name\n" + + "CREATE FUNCTION \"name\"(x Location) RETURNS string AS x.name\n" + "CREATE FUNCTION id(x bigint) RETURNS bigint AS x"; try (var ddl = Ddl.builder().database(URI.create("/TEST/QT")).relationalExtension(relationalExtension).schemaTemplate(schemaTemplate).build()) { From 8076737786afb320215383109162d2f6799a4862 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Mon, 3 Feb 2025 10:22:09 -0800 Subject: [PATCH 35/41] save --- .../recordlayer/query/visitors/ExpressionVisitor.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index 6b15f0bb6d..8e1b3192d4 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -115,7 +115,6 @@ public Expression visitFullColumnNameExpressionAtom(@Nonnull RelationalParser.Fu @Nonnull @Override public Expressions visitSelectElements(@Nonnull RelationalParser.SelectElementsContext selectElementsContext) { - System.out.println("visitSelectElements is called"); return Expressions.of(selectElementsContext.selectElement().stream() .map(selectElement -> Assert.castUnchecked(selectElement.accept(this), Expression.class)) .collect(ImmutableList.toImmutableList())); @@ -251,7 +250,6 @@ public Expression visitCaseFunctionCall(@Nonnull RelationalParser.CaseFunctionCa @Nonnull @Override public Expression visitUserDefinedFunctionCall(@Nonnull RelationalParser.UserDefinedFunctionCallContext ctx) { - System.out.println("visitUserDefinedFunctionCall called"); final var functionName = ctx.userDefinedFunctionName().getText(); Expressions arguments = visitFunctionArgs(ctx.functionArgs()); return getDelegate().resolveFunction(functionName, arguments.asList().toArray(new Expression[0])); @@ -702,7 +700,6 @@ public Expression visitArrayConstructor(@Nonnull RelationalParser.ArrayConstruct @Nonnull private Expressions parseRecordFields(@Nonnull List parserRuleContexts, @Nullable List targetFields) { - System.out.println("parseRecordFields is called"); Assert.thatUnchecked(targetFields == null || targetFields.size() == parserRuleContexts.size()); final var resultsBuilder = ImmutableList.builder(); for (int i = 0; i < parserRuleContexts.size(); i++) { From cea7e31bbddb926d0e90bc8016c7d4844a5e88b5 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Mon, 3 Feb 2025 12:09:24 -0800 Subject: [PATCH 36/41] clean --- .../relational/recordlayer/metadata/RecordLayerTable.java | 1 + .../relational/recordlayer/query/SemanticAnalyzer.java | 8 +++----- .../recordlayer/query/functions/SqlFunctionCatalog.java | 1 + .../relational/recordlayer/query/visitors/DdlVisitor.java | 3 ++- .../foundationdb/relational/api/DriverManagerTest.java | 4 ++-- .../relational/recordlayer/query/StandardQueryTests.java | 2 -- gradle/strict.gradle | 1 - 7 files changed, 9 insertions(+), 11 deletions(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java index 12ce0cf0d3..5a9170eb3a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerTable.java @@ -337,6 +337,7 @@ public RecordLayerTable build() { Assert.thatUnchecked(!columnsList.isEmpty(), ErrorCode.INVALID_TABLE_DEFINITION, "Attempting to create table %s without columns", name); final var indexesSet = ImmutableSet.copyOf(indexes); + return new RecordLayerTable(name, columnsList, indexesSet, getPrimaryKey(), generations, dataType, record); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index 973ea46de1..99d02c1b36 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -366,7 +366,7 @@ private List lookup(@Nonnull Identifier referenceIdentifier, continue; } } - final var nestedFieldMaybe = lookUpNestedField(referenceIdentifier, attribute, operator, matchQualifiedOnly); + final var nestedFieldMaybe = lookupNestedField(referenceIdentifier, attribute, operator, matchQualifiedOnly); if (nestedFieldMaybe.isPresent()) { matchedAttributes.add(nestedFieldMaybe.get()); checkForPseudoColumns = false; @@ -406,7 +406,7 @@ public Optional lookupAlias(@Nonnull Identifier requestedAlias, } @Nonnull - public Optional lookUpNestedField(@Nonnull Identifier requestedIdentifier, + public Optional lookupNestedField(@Nonnull Identifier requestedIdentifier, @Nonnull Expression existingExpression, @Nonnull LogicalOperator logicalOperator, boolean matchQualifiedOnly) { @@ -461,7 +461,7 @@ public Optional lookUpNestedField(@Nonnull Identifier requestedIdent } @Nonnull - public Optional lookUpNestedField(@Nonnull Identifier requestedIdentifier, + public Optional lookupNestedField(@Nonnull Identifier requestedIdentifier, @Nonnull Identifier paramId, @Nonnull QuantifiedObjectValue existingValue, @Nonnull DataType targetDataType) { @@ -494,7 +494,6 @@ public Optional lookUpNestedField(@Nonnull Identifier requestedIdentifier return Optional.empty(); } } - // probably need to check if currentDataType = targetDataType final var fieldPath = FieldValue.resolveFieldPath(existingValue.getResultType(), accessors.build()); final var fieldValue = FieldValue.ofFieldsAndFuseIfPossible(existingValue, fieldPath); Assert.thatUnchecked(fieldValue.getResultType().equals(DataTypeUtils.toRecordLayerType(targetDataType)), ErrorCode.DATATYPE_MISMATCH, "Result data types don't match!"); @@ -766,7 +765,6 @@ public static void validateContinuation(@Nonnull Expression continuation) { @Nonnull public Expression resolveFunction(@Nonnull String functionName, boolean flattenSingleItemRecords, @Nonnull Expression... arguments) { - System.out.println("functionName:" + functionName); Assert.thatUnchecked(functionCatalog.containsFunction(functionName), ErrorCode.UNSUPPORTED_QUERY, () -> String.format("Unsupported operator %s", functionName)); final var builtInFunction = functionCatalog.lookUpFunction(functionName); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java index 2ee090eac0..61bd178b29 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.relational.recordlayer.query.functions; import com.apple.foundationdb.annotation.API; + import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; import com.apple.foundationdb.record.query.plan.cascades.Function; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 0e462d6614..519171bb6c 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.relational.recordlayer.query.visitors; import com.apple.foundationdb.annotation.API; + import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.MacroFunction; @@ -198,7 +199,7 @@ public ScalarValuedFunction visitFunctionDefinition(@Nonnull RelationalParser.Fu final var functionBody = visitFullId(ctx.fullId()); final var paramNameId = Identifier.of(ctx.paramName.getText().toUpperCase(Locale.ROOT)); - Optional fieldValue = semanticAnalyzer.lookUpNestedField(functionBody, paramNameId, argumentValue, returnType); + Optional fieldValue = semanticAnalyzer.lookupNestedField(functionBody, paramNameId, argumentValue, returnType); Assert.thatUnchecked(fieldValue.isPresent(), "couldn't resolve function definition"); return new ScalarValuedFunction(new MacroFunction(ctx.functionName.getText(), List.of(argumentValue), fieldValue.get())); diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java index 82475c98e4..18127236dc 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.relational.api; +import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -29,7 +30,6 @@ import java.sql.DriverManager; import java.sql.SQLException; -import java.util.stream.Collectors; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; @@ -44,7 +44,7 @@ class DriverManagerTest { @BeforeEach public void beforeEach() throws SQLException { // Cleanup old drivers, if any, - final var oldDrivers = DriverManager.drivers().collect(Collectors.toList()); + final var oldDrivers = DriverManager.drivers().collect(ImmutableList.toImmutableList()); for (final var driver: oldDrivers) { DriverManager.deregisterDriver(driver); } diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java index 1f933205e5..97a4ac48aa 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/recordlayer/query/StandardQueryTests.java @@ -31,7 +31,6 @@ import com.apple.foundationdb.relational.api.RelationalStructMetaData; import com.apple.foundationdb.relational.api.exceptions.ContextualSQLException; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; -import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.recordlayer.ContinuationImpl; import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalExtension; import com.apple.foundationdb.relational.recordlayer.EmbeddedRelationalStatement; @@ -52,7 +51,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.net.URI; -import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.sql.Array; import java.sql.Connection; diff --git a/gradle/strict.gradle b/gradle/strict.gradle index cfefcb334b..5b2366b274 100644 --- a/gradle/strict.gradle +++ b/gradle/strict.gradle @@ -23,7 +23,6 @@ tasks.withType(JavaCompile) { // suppress static due to protoc generating super.addAll calls. options.compilerArgs += ["-Xlint:all", "-Xlint:-processing", "-Xlint:-options", "-Xlint:-static", "-Werror"] - options.compilerArgs.remove('-Werror') } spotbugs { From a71c70b53f3e7c42bad40515e5dbb5e9a303efa1 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Tue, 4 Feb 2025 13:30:20 -0800 Subject: [PATCH 37/41] save --- .../main/java/com/apple/foundationdb/record/RecordMetaData.java | 1 - .../foundationdb/record/metadata/ScalarValuedFunction.java | 2 +- .../apple/foundationdb/record/query/plan/cascades/Function.java | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java index 999eb9e124..44a9484f86 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java @@ -704,7 +704,6 @@ public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor builder.addFormerIndexes(formerIndex.toProto()); } - // Add in the final options. builder.addAllScalarValuedFunction(scalarValuedFunctions.stream().map(ScalarValuedFunction::toProto).collect(Collectors.toList())); builder.setSplitLongRecords(splitLongRecords); builder.setStoreRecordVersions(storeRecordVersions); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/ScalarValuedFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/ScalarValuedFunction.java index 924cfbc002..0c671e954b 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/ScalarValuedFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/ScalarValuedFunction.java @@ -1,5 +1,5 @@ /* - * UDF.java + * ScalarValuedFunction.java * * This source file is part of the FoundationDB open source project * diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java index 131de81195..f35bf63ee4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java @@ -1,5 +1,5 @@ /* - * BuiltInFunction.java + * Function.java * * This source file is part of the FoundationDB open source project * From 7f030c94b12f40e028567ce89da25b5953f5fb07 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 20 Mar 2025 13:51:33 -0700 Subject: [PATCH 38/41] checkstyle --- .../foundationdb/record/RecordMetaData.java | 12 ++-- .../record/RecordMetaDataBuilder.java | 6 +- .../record/metadata/ScalarValuedFunction.java | 67 ------------------- .../recordlayer/CatalogMetaData.java | 1 - .../metadata/RecordLayerSchemaTemplate.java | 33 +++++---- .../serde/RecordMetadataDeserializer.java | 4 +- .../serde/RecordMetadataSerializer.java | 2 +- .../recordlayer/query/SemanticAnalyzer.java | 7 +- .../query/functions/SqlFunctionCatalog.java | 8 ++- .../functions/SqlFunctionCatalogImpl.java | 23 ++++++- .../query/visitors/DdlVisitor.java | 9 ++- 11 files changed, 60 insertions(+), 112 deletions(-) delete mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/ScalarValuedFunction.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java index efaf23220d..7e3debf1c3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java @@ -28,7 +28,6 @@ import com.apple.foundationdb.record.metadata.MetaDataException; import com.apple.foundationdb.record.metadata.RecordType; import com.apple.foundationdb.record.metadata.SyntheticRecordType; -import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.metadata.UnnestedRecordType; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression; @@ -51,7 +50,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.TreeMap; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -199,6 +197,11 @@ public Map getRecordTypes() { return recordTypes; } + @Nonnull + public Map getUserDefinedFunctionMap() { + return userDefinedFunctionMap; + } + @Nonnull public RecordType getRecordType(@Nonnull String name) { RecordType recordType = recordTypes.get(name); @@ -351,11 +354,6 @@ public List getFormerIndexes() { return formerIndexes; } - @Nonnull - public Collection getAllScalarValuedFunctions() { - return scalarValuedFunctions; - } - public boolean isSplitLongRecords() { return splitLongRecords; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java index d30e288060..79836df6b6 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java @@ -35,7 +35,6 @@ import com.apple.foundationdb.record.metadata.RecordTypeIndexesBuilder; import com.apple.foundationdb.record.metadata.SyntheticRecordType; import com.apple.foundationdb.record.metadata.SyntheticRecordTypeBuilder; -import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.metadata.UnnestedRecordTypeBuilder; import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression; import com.apple.foundationdb.record.metadata.expressions.KeyExpression; @@ -229,9 +228,10 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey())); } } + PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, + PlanHashable.CURRENT_FOR_CONTINUATION); for (RecordMetaDataProto.PUserDefinedFunction function: metaDataProto.getUserDefinedFunctionsList()) { - UserDefinedFunction func = (UserDefinedFunction)PlanSerialization.dispatchFromProtoContainer(new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, - PlanHashable.CURRENT_FOR_CONTINUATION), function); + UserDefinedFunction func = (UserDefinedFunction)PlanSerialization.dispatchFromProtoContainer(serializationContext, function); userDefinedFunctionMap.put(func.getFunctionName(), func); } if (metaDataProto.hasSplitLongRecords()) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/ScalarValuedFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/ScalarValuedFunction.java deleted file mode 100644 index 0c671e954b..0000000000 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/ScalarValuedFunction.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * ScalarValuedFunction.java - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apple.foundationdb.record.metadata; - -import com.apple.foundationdb.record.PlanHashable; -import com.apple.foundationdb.record.PlanSerializationContext; -import com.apple.foundationdb.record.RecordMetaDataProto; -import com.apple.foundationdb.record.query.plan.cascades.MacroFunction; -import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry; - -import javax.annotation.Nonnull; - -/** - * Defines a scalar User-defined-function. - */ -public class ScalarValuedFunction { - @Nonnull - private final MacroFunction macroFunction; - - public ScalarValuedFunction(@Nonnull MacroFunction functionValue) { - this.macroFunction = functionValue; - } - - @Nonnull - public MacroFunction getMacroFunction() { - return macroFunction; - } - - @Nonnull - public String getFunctionName() { - return macroFunction.getFunctionName(); - } - - @Nonnull - public RecordMetaDataProto.ScalarValuedFunction toProto() { - PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, - PlanHashable.CURRENT_FOR_CONTINUATION); - return RecordMetaDataProto.ScalarValuedFunction.newBuilder() - .setFunctionValue(macroFunction.toProto(serializationContext)) - .build(); - } - - @Nonnull - public static ScalarValuedFunction fromProto(RecordMetaDataProto.ScalarValuedFunction proto) { - PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE, - PlanHashable.CURRENT_FOR_CONTINUATION); - return new ScalarValuedFunction(MacroFunction.fromProto(serializationContext, proto.getFunctionValue())); - } -} diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java index a8e7a51783..2a4a901315 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java @@ -25,7 +25,6 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; -import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.MetaDataException; import com.apple.foundationdb.record.metadata.RecordType; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java index 1caea53bf2..dbe8b66fde 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java @@ -25,8 +25,8 @@ import com.apple.foundationdb.record.RecordMetaData; import com.apple.foundationdb.record.RecordMetaDataProto; import com.apple.foundationdb.record.metadata.Key; -import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.query.combinatorics.TopologicalSort; +import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.exceptions.RelationalException; import com.apple.foundationdb.relational.api.metadata.DataType; @@ -54,7 +54,6 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -71,7 +70,7 @@ public final class RecordLayerSchemaTemplate implements SchemaTemplate { private final Set tables; @Nonnull - private final Set scalarValuedFunctions; + private final Set userDefinedFunctions; private final int version; @@ -90,13 +89,13 @@ public final class RecordLayerSchemaTemplate implements SchemaTemplate { private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, - @Nonnull final Set scalarValuedFunctions, + @Nonnull final Set userDefinedFunctions, int version, boolean enableLongRows, boolean storeRowVersions) { this.name = name; this.tables = tables; - this.scalarValuedFunctions = scalarValuedFunctions; + this.userDefinedFunctions = userDefinedFunctions; this.version = version; this.enableLongRows = enableLongRows; this.storeRowVersions = storeRowVersions; @@ -107,7 +106,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, - @Nonnull final Set scalarValuedFunctions, + @Nonnull final Set userDefinedFunctions, int version, boolean enableLongRows, boolean storeRowVersions, @@ -115,7 +114,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, this.name = name; this.version = version; this.tables = tables; - this.scalarValuedFunctions = scalarValuedFunctions; + this.userDefinedFunctions = userDefinedFunctions; this.enableLongRows = enableLongRows; this.storeRowVersions = storeRowVersions; this.metaDataSupplier = Suppliers.memoize(() -> cachedMetadata); @@ -157,8 +156,8 @@ public RecordLayerSchema generateSchema(@Nonnull String databaseId, @Nonnull Str } @Nonnull - public Set getAllScalarValuedFunctions() { - return scalarValuedFunctions; + public Set getAllUserDefinedFunctions() { + return userDefinedFunctions; } @Nonnull @@ -323,7 +322,7 @@ public static final class Builder { private final Map tables; - private final Set scalarValuedFunctions; + private final Set userDefinedFunctionSet; private final Map auxiliaryTypes; // for quick lookup @@ -331,7 +330,7 @@ public static final class Builder { private Builder() { tables = new LinkedHashMap<>(); - scalarValuedFunctions = new HashSet<>(); + userDefinedFunctionSet = new HashSet<>(); auxiliaryTypes = new LinkedHashMap<>(); // enable long rows is TRUE by default enableLongRows = true; @@ -420,14 +419,14 @@ public Builder addAuxiliaryTypes(@Nonnull Collection auxiliaryTy } @Nonnull - public Builder addScalarValuedFunction(@Nonnull ScalarValuedFunction scalarValuedFunction) { - scalarValuedFunctions.add(scalarValuedFunction); + public Builder addUserDefinedFunction(@Nonnull UserDefinedFunction userDefinedFunction) { + userDefinedFunctionSet.add(userDefinedFunction); return this; } @Nonnull - public Builder addScalarValuedFunctions(@Nonnull Collection scalarValuedFunctions) { - scalarValuedFunctions.forEach(this::addScalarValuedFunction); + public Builder addUserDefinedFunctions(@Nonnull Collection functions) { + functions.forEach(this::addUserDefinedFunction); return this; } @@ -489,9 +488,9 @@ public RecordLayerSchemaTemplate build() { resolveTypes(); } if (cachedMetadata != null) { - return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), scalarValuedFunctions, version, enableLongRows, storeRowVersions, cachedMetadata); + return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), userDefinedFunctionSet, version, enableLongRows, storeRowVersions, cachedMetadata); } else { - return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), scalarValuedFunctions, version, enableLongRows, storeRowVersions); + return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), userDefinedFunctionSet, version, enableLongRows, storeRowVersions); } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java index 76bc28bfdc..1db60804f8 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataDeserializer.java @@ -61,8 +61,8 @@ public RecordLayerSchemaTemplate.Builder getSchemaTemplate(@Nonnull final String .setEnableLongRows(recordMetaData.isSplitLongRecords()) .setName(schemaTemplateName) .setIntermingleTables(!recordMetaData.primaryKeyHasRecordTypePrefix()); - for (final var u: recordMetaData.getAllScalarValuedFunctions()) { - schemaTemplateBuilder.addScalarValuedFunction(u); + for (final var u: recordMetaData.getUserDefinedFunctionMap().values()) { + schemaTemplateBuilder.addUserDefinedFunction(u); } final var nameToTableBuilder = new HashMap(); for (final var registeredType : registeredTypes) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java index 94690eabe4..e31f834517 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/serde/RecordMetadataSerializer.java @@ -87,7 +87,7 @@ public void visit(@Nonnull com.apple.foundationdb.relational.api.metadata.Index @Override public void visit(@Nonnull SchemaTemplate schemaTemplate) { Assert.thatUnchecked(schemaTemplate instanceof RecordLayerSchemaTemplate); - getBuilder().addScalarValuedFunctions(((RecordLayerSchemaTemplate) schemaTemplate).getAllScalarValuedFunctions()); + getBuilder().addUserDefinedFunctions(((RecordLayerSchemaTemplate) schemaTemplate).getAllUserDefinedFunctions()); getBuilder().setSplitLongRecords(schemaTemplate.isEnableLongRows()); getBuilder().setStoreRecordVersions(schemaTemplate.isStoreRowVersions()); getBuilder().setVersion(schemaTemplate.getVersion()); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index 4c0d6199e0..dee661234f 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -21,13 +21,13 @@ package com.apple.foundationdb.relational.recordlayer.query; import com.apple.foundationdb.annotation.API; -import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.query.plan.cascades.AccessHint; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; import com.apple.foundationdb.record.query.plan.cascades.Correlated; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.IndexAccessHint; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; import com.apple.foundationdb.record.query.plan.cascades.values.AggregateValue; @@ -38,6 +38,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.IndexableAggregateValue; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; import com.apple.foundationdb.record.query.plan.cascades.values.NotValue; +import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.RelOpValue; import com.apple.foundationdb.record.query.plan.cascades.values.StreamableAggregateValue; @@ -102,8 +103,8 @@ public SemanticAnalyzer(@Nonnull SchemaTemplate metadataCatalog, this.metadataCatalog = metadataCatalog; this.functionCatalog = functionCatalog; // add UDFs to functionCatalog - for (ScalarValuedFunction scalarValuedFunction: ((RecordLayerSchemaTemplate) metadataCatalog).getAllScalarValuedFunctions()) { - ((SqlFunctionCatalog)this.functionCatalog).addFunction(scalarValuedFunction); + for (UserDefinedFunction function: ((RecordLayerSchemaTemplate) metadataCatalog).getAllUserDefinedFunctions()) { + (this.functionCatalog).addUdfFunction(function); } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java index 47d09261dc..2cc8535239 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java @@ -20,17 +20,19 @@ package com.apple.foundationdb.relational.recordlayer.query.functions; -import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; -import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; +import com.apple.foundationdb.record.query.plan.cascades.CatalogedFunction; +import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; import com.apple.foundationdb.relational.recordlayer.query.Expression; import javax.annotation.Nonnull; public interface SqlFunctionCatalog { @Nonnull - BuiltInFunction lookUpFunction(@Nonnull String name, @Nonnull Expression... expressions); + CatalogedFunction lookUpFunction(@Nonnull String name, @Nonnull Expression... expressions); boolean containsFunction(@Nonnull String name); boolean isUdfFunction(@Nonnull String name); + + void addUdfFunction(@Nonnull UserDefinedFunction function); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java index 31ee33c108..ff83f8f187 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java @@ -23,6 +23,8 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; +import com.apple.foundationdb.record.query.plan.cascades.CatalogedFunction; +import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; import com.apple.foundationdb.record.query.plan.cascades.values.FunctionCatalog; import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue; @@ -33,7 +35,9 @@ import com.google.common.collect.ImmutableMap; import javax.annotation.Nonnull; +import java.util.HashMap; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.function.Function; import java.util.stream.StreamSupport; @@ -52,19 +56,27 @@ public final class SqlFunctionCatalogImpl implements SqlFunctionCatalog { @Nonnull private final ImmutableMap>> synonyms; + @Nonnull + private final Map userDefinedFunctionMap = new HashMap<>(); + + private SqlFunctionCatalogImpl() { this.synonyms = createSynonyms(); } @Nonnull @Override - public BuiltInFunction lookUpFunction(@Nonnull final String name, @Nonnull final Expression... expressions) { - return Assert.notNullUnchecked(Objects.requireNonNull(synonyms.get(name.toLowerCase(Locale.ROOT))).apply(expressions.length)); + public CatalogedFunction lookUpFunction(@Nonnull final String name, @Nonnull final Expression... expressions) { + if (synonyms.get(name.toLowerCase(Locale.ROOT)) != null) { + return Assert.notNullUnchecked(Objects.requireNonNull(synonyms.get(name.toLowerCase(Locale.ROOT))).apply(expressions.length)); + } else { + return userDefinedFunctionMap.get(name.toLowerCase(Locale.ROOT)); + } } @Override public boolean containsFunction(@Nonnull String name) { - return synonyms.containsKey(name.toLowerCase(Locale.ROOT)); + return synonyms.containsKey(name.toLowerCase(Locale.ROOT)) || userDefinedFunctionMap.containsKey(name.toLowerCase(Locale.ROOT)); } @Override @@ -72,6 +84,11 @@ public boolean isUdfFunction(@Nonnull final String name) { return "java_call".equals(name.trim().toLowerCase(Locale.ROOT)); } + @Override + public void addUdfFunction(@Nonnull final UserDefinedFunction function) { + userDefinedFunctionMap.put(function.getFunctionName(), function); + } + @Nonnull private static ImmutableMap>> createSynonyms() { return ImmutableMap.>>builder() diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java index 3933b34a38..048cccd413 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DdlVisitor.java @@ -22,9 +22,9 @@ import com.apple.foundationdb.annotation.API; -import com.apple.foundationdb.record.metadata.ScalarValuedFunction; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.MacroFunction; +import com.apple.foundationdb.record.query.plan.cascades.UserDefinedFunction; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalSortExpression; import com.apple.foundationdb.record.query.plan.cascades.values.QuantifiedObjectValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; @@ -186,7 +186,7 @@ public RecordLayerIndex visitIndexDefinition(@Nonnull RelationalParser.IndexDefi @Nonnull @Override - public ScalarValuedFunction visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { + public UserDefinedFunction visitFunctionDefinition(@Nonnull RelationalParser.FunctionDefinitionContext ctx) { final var ddlCatalog = metadataBuilder.build(); // parse the function definition using the newly constructed metadata. getDelegate().replaceCatalog(ddlCatalog); @@ -206,8 +206,7 @@ public ScalarValuedFunction visitFunctionDefinition(@Nonnull RelationalParser.Fu Optional fieldValue = semanticAnalyzer.lookupNestedField(functionBody, paramNameId, argumentValue, returnType); Assert.thatUnchecked(fieldValue.isPresent(), "couldn't resolve function definition"); - - return new ScalarValuedFunction(new MacroFunction(ctx.functionName.getText(), List.of(argumentValue), fieldValue.get())); + return new MacroFunction(ctx.functionName.getText(), List.of(argumentValue), fieldValue.get()); } @Nonnull @@ -270,7 +269,7 @@ public ProceduralPlan visitCreateSchemaTemplateStatement(@Nonnull RelationalPars metadataBuilder.addTable(tableWithIndex); } final var udfs = functionClauses.build().stream().map(this::visitFunctionDefinition).collect(ImmutableList.toImmutableList()); - metadataBuilder.addScalarValuedFunctions(udfs); + metadataBuilder.addUserDefinedFunctions(udfs); return ProceduralPlan.of(metadataOperationsFactory.getCreateSchemaTemplateConstantAction(metadataBuilder.build(), Options.NONE)); } From e3bbc51dcd5adde11320f339017784d50b8979b0 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 20 Mar 2025 14:14:50 -0700 Subject: [PATCH 39/41] clean --- .../query/plan/cascades/BuiltInFunction.java | 2 +- .../record/query/plan/cascades/Function.java | 121 ------------------ .../cascades/values/JavaCallFunction.java | 2 +- .../plan/cascades/values/UdfFunction.java | 2 +- .../plan/cascades/MacroFunctionTest.java | 1 - .../recordlayer/CatalogMetaData.java | 2 +- .../recordlayer/query/SemanticAnalyzer.java | 1 + .../query/functions/SqlFunctionCatalog.java | 2 +- .../query/visitors/ExpressionVisitor.java | 1 + .../relational/api/DriverManagerTest.java | 2 +- 10 files changed, 8 insertions(+), 128 deletions(-) delete mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java index 3ef0535c17..f6b3dcf712 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java @@ -48,7 +48,7 @@ public abstract class BuiltInFunction extends CatalogedFunction * @param parameterTypes The type of the parameter(s). * @param encapsulationFunction An encapsulation of the function's runtime computation. */ - public BuiltInFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final EncapsulationFunction encapsulationFunction) { + protected BuiltInFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final EncapsulationFunction encapsulationFunction) { super(functionName, parameterTypes, null); this.encapsulationFunction = encapsulationFunction; } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java deleted file mode 100644 index f35bf63ee4..0000000000 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/Function.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Function.java - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2015-2022 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.apple.foundationdb.record.query.plan.cascades; - -import com.apple.foundationdb.record.query.plan.cascades.typing.Type; -import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; -import com.google.common.base.Verify; -import com.google.common.collect.ImmutableList; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.util.List; -import java.util.stream.Collectors; - -/** - * Main interface for defining a function that can be evaluated against a number of arguments. - * @param The resulting type of the function. - */ -@SuppressWarnings("PMD.AbstractClassWithoutAbstractMethod") -public abstract class Function { - @Nonnull - final String functionName; - - @Nonnull - final List parameterTypes; - - /** - * The type of the function's variadic parameters (if any). - */ - @Nullable - final Type variadicSuffixType; - - /** - * Creates a new instance of {@link Function}. - * @param functionName The name of the function. - * @param parameterTypes The type of the parameter(s). - * @param variadicSuffixType The type of the function's vararg. - */ - public Function(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nullable final Type variadicSuffixType) { - this.functionName = functionName; - this.parameterTypes = ImmutableList.copyOf(parameterTypes); - this.variadicSuffixType = variadicSuffixType; - } - - @Nonnull - public String getFunctionName() { - return functionName; - } - - @Nonnull - public List getParameterTypes() { - return parameterTypes; - } - - @Nonnull - public Type resolveParameterType(int index) { - Verify.verify(index >= 0, "unexpected negative parameter index"); - if (index < parameterTypes.size()) { - return parameterTypes.get(index); - } else { - if (hasVariadicSuffix()) { - return variadicSuffixType; - } - throw new IllegalArgumentException("cannot resolve declared parameter at index " + index); - } - } - - @Nonnull - public List resolveParameterTypes(int numberOfArguments) { - Verify.verify(numberOfArguments > 0, "unexpected number of arguments"); - final ImmutableList.Builder resultBuilder = ImmutableList.builder(); - for (int i = 0; i < numberOfArguments; i ++) { - resultBuilder.add(resolveParameterType(i)); - } - return resultBuilder.build(); - } - - @Nullable - public Type getVariadicSuffixType() { - return variadicSuffixType; - } - - public boolean hasVariadicSuffix() { - return variadicSuffixType != null; - } - - @Nonnull - public abstract T encapsulate(@Nonnull List arguments); - - @Nonnull - @Override - public String toString() { - String variadicSuffixString = ""; - if (variadicSuffixType != null) { - variadicSuffixString = variadicSuffixType + "..."; - if (!parameterTypes.isEmpty()) { - variadicSuffixString = ", " + variadicSuffixString; - } - } - - return functionName + "(" + parameterTypes.stream().map(Object::toString).collect(Collectors.joining(",")) + variadicSuffixString + ")"; - } -} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/JavaCallFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/JavaCallFunction.java index 4b2094c729..407985f624 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/JavaCallFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/JavaCallFunction.java @@ -72,7 +72,7 @@ private static Value findFunction(@Nonnull final BuiltInFunction ignored, // the class must have parameterless constructor try { final Constructor constructor = clazz.getDeclaredConstructor(); - return ((UdfFunction)constructor.newInstance()).encapsulate(arguments.stream().skip(1).collect(Collectors.toUnmodifiableList())); + return (Value)((UdfFunction)constructor.newInstance()).encapsulate(arguments.stream().skip(1).collect(Collectors.toUnmodifiableList())); } catch (NoSuchMethodException | InstantiationException | IllegalAccessException | InvocationTargetException e) { throw new RecordCoreException("could not instantiate call-site from '" + clazz.getName() + "'", e); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/UdfFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/UdfFunction.java index da83bd29fc..6f922f613e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/UdfFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/UdfFunction.java @@ -56,7 +56,7 @@ public final String getFunctionName() { @Nonnull @Override - public final Value encapsulate(@Nonnull final List arguments) { + public final Typed encapsulate(@Nonnull final List arguments) { arguments.forEach(argument -> Verify.verify(argument instanceof Value)); final List parameterTypes = getParameterTypes(); if (arguments.size() != parameterTypes.size()) { diff --git a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunctionTest.java b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunctionTest.java index 321e990e80..0ff371ca33 100644 --- a/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunctionTest.java +++ b/fdb-record-layer-core/src/test/java/com/apple/foundationdb/record/query/plan/cascades/MacroFunctionTest.java @@ -64,7 +64,6 @@ void testAdd() { QuantifiedObjectValue param2 = QuantifiedObjectValue.of(CorrelationIdentifier.uniqueID(), record); ArithmeticValue bodyValue = new ArithmeticValue(ArithmeticValue.PhysicalOperator.ADD_LL, param1, param2); - MacroFunction macroFunction = new MacroFunction("add", ImmutableList.of(param1, param2), bodyValue); RecordConstructorValue testValue1 = RecordConstructorValue.ofColumns(ImmutableList.of( diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java index 2a4a901315..3e14a5abbc 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/CatalogMetaData.java @@ -23,8 +23,8 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.RecordMetaData; -import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.RecordMetaDataProto; +import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto; import com.apple.foundationdb.record.metadata.Index; import com.apple.foundationdb.record.metadata.MetaDataException; import com.apple.foundationdb.record.metadata.RecordType; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index dee661234f..6ebe14d214 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.relational.recordlayer.query; import com.apple.foundationdb.annotation.API; + import com.apple.foundationdb.record.query.plan.cascades.AccessHint; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; import com.apple.foundationdb.record.query.plan.cascades.Correlated; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java index 2cc8535239..fcfed9362c 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalog.java @@ -33,6 +33,6 @@ public interface SqlFunctionCatalog { boolean containsFunction(@Nonnull String name); boolean isUdfFunction(@Nonnull String name); - + void addUdfFunction(@Nonnull UserDefinedFunction function); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index cd8d563361..a90172e48e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -21,6 +21,7 @@ package com.apple.foundationdb.relational.recordlayer.query.visitors; import com.apple.foundationdb.annotation.API; + import com.apple.foundationdb.record.query.plan.cascades.Quantifier; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.AbstractArrayConstructorValue; diff --git a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java index 18127236dc..1e7a2dca9a 100644 --- a/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java +++ b/fdb-relational-core/src/test/java/com/apple/foundationdb/relational/api/DriverManagerTest.java @@ -45,7 +45,7 @@ class DriverManagerTest { public void beforeEach() throws SQLException { // Cleanup old drivers, if any, final var oldDrivers = DriverManager.drivers().collect(ImmutableList.toImmutableList()); - for (final var driver: oldDrivers) { + for (final var driver : oldDrivers) { DriverManager.deregisterDriver(driver); } DriverManager.drivers().forEach(driver -> { From f5256d9a3aea18f1a64a40944e8f18c224f8a654 Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Thu, 20 Mar 2025 14:20:07 -0700 Subject: [PATCH 40/41] nit --- .idea/compiler.xml | 41 ------------------- .../query/plan/cascades/BuiltInFunction.java | 3 +- .../metadata/RecordLayerSchemaTemplate.java | 25 +++++------ 3 files changed, 14 insertions(+), 55 deletions(-) delete mode 100644 .idea/compiler.xml diff --git a/.idea/compiler.xml b/.idea/compiler.xml deleted file mode 100644 index 2a53fe5b6f..0000000000 --- a/.idea/compiler.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java index f6b3dcf712..950fc6b187 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInFunction.java @@ -49,8 +49,7 @@ public abstract class BuiltInFunction extends CatalogedFunction * @param encapsulationFunction An encapsulation of the function's runtime computation. */ protected BuiltInFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, @Nonnull final EncapsulationFunction encapsulationFunction) { - super(functionName, parameterTypes, null); - this.encapsulationFunction = encapsulationFunction; + this(functionName, parameterTypes, null, encapsulationFunction); } /** diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java index dbe8b66fde..c30e994c88 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/metadata/RecordLayerSchemaTemplate.java @@ -51,6 +51,7 @@ import javax.annotation.Nonnull; import java.util.BitSet; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -70,7 +71,7 @@ public final class RecordLayerSchemaTemplate implements SchemaTemplate { private final Set tables; @Nonnull - private final Set userDefinedFunctions; + private final Map userDefinedFunctionMap; private final int version; @@ -89,13 +90,13 @@ public final class RecordLayerSchemaTemplate implements SchemaTemplate { private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, - @Nonnull final Set userDefinedFunctions, + @Nonnull final Map userDefinedFunctionMap, int version, boolean enableLongRows, boolean storeRowVersions) { this.name = name; this.tables = tables; - this.userDefinedFunctions = userDefinedFunctions; + this.userDefinedFunctionMap = userDefinedFunctionMap; this.version = version; this.enableLongRows = enableLongRows; this.storeRowVersions = storeRowVersions; @@ -106,7 +107,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, private RecordLayerSchemaTemplate(@Nonnull final String name, @Nonnull final Set tables, - @Nonnull final Set userDefinedFunctions, + @Nonnull final Map userDefinedFunctionMap, int version, boolean enableLongRows, boolean storeRowVersions, @@ -114,7 +115,7 @@ private RecordLayerSchemaTemplate(@Nonnull final String name, this.name = name; this.version = version; this.tables = tables; - this.userDefinedFunctions = userDefinedFunctions; + this.userDefinedFunctionMap = userDefinedFunctionMap; this.enableLongRows = enableLongRows; this.storeRowVersions = storeRowVersions; this.metaDataSupplier = Suppliers.memoize(() -> cachedMetadata); @@ -156,8 +157,8 @@ public RecordLayerSchema generateSchema(@Nonnull String databaseId, @Nonnull Str } @Nonnull - public Set getAllUserDefinedFunctions() { - return userDefinedFunctions; + public Collection getAllUserDefinedFunctions() { + return userDefinedFunctionMap.values(); } @Nonnull @@ -322,7 +323,7 @@ public static final class Builder { private final Map tables; - private final Set userDefinedFunctionSet; + private final Map functionMap; private final Map auxiliaryTypes; // for quick lookup @@ -330,7 +331,7 @@ public static final class Builder { private Builder() { tables = new LinkedHashMap<>(); - userDefinedFunctionSet = new HashSet<>(); + functionMap = new HashMap<>(); auxiliaryTypes = new LinkedHashMap<>(); // enable long rows is TRUE by default enableLongRows = true; @@ -420,7 +421,7 @@ public Builder addAuxiliaryTypes(@Nonnull Collection auxiliaryTy @Nonnull public Builder addUserDefinedFunction(@Nonnull UserDefinedFunction userDefinedFunction) { - userDefinedFunctionSet.add(userDefinedFunction); + functionMap.put(userDefinedFunction.getFunctionName(), userDefinedFunction); return this; } @@ -488,9 +489,9 @@ public RecordLayerSchemaTemplate build() { resolveTypes(); } if (cachedMetadata != null) { - return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), userDefinedFunctionSet, version, enableLongRows, storeRowVersions, cachedMetadata); + return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), functionMap, version, enableLongRows, storeRowVersions, cachedMetadata); } else { - return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), userDefinedFunctionSet, version, enableLongRows, storeRowVersions); + return new RecordLayerSchemaTemplate(name, new LinkedHashSet<>(tables.values()), functionMap, version, enableLongRows, storeRowVersions); } } From 684d5fca18af3aa94a3f8dc2c4b4546320a02e8b Mon Sep 17 00:00:00 2001 From: Pengpeng Lu Date: Mon, 31 Mar 2025 13:42:13 -0700 Subject: [PATCH 41/41] style --- .../recordlayer/query/functions/SqlFunctionCatalogImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java index ff83f8f187..3d148ea353 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java @@ -51,7 +51,7 @@ public final class SqlFunctionCatalogImpl implements SqlFunctionCatalog { @Nonnull - private static final SqlFunctionCatalogImpl INSTANCE = new SqlFunctionCatalogImpl(); + public static final SqlFunctionCatalogImpl INSTANCE = new SqlFunctionCatalogImpl(); @Nonnull private final ImmutableMap>> synonyms;