diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInTableFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInTableFunction.java new file mode 100644 index 0000000000..9b3aad3c09 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInTableFunction.java @@ -0,0 +1,41 @@ +/* + * BuiltInTableFunction.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades; + +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +public abstract class BuiltInTableFunction extends BuiltInFunction { + protected BuiltInTableFunction(@Nonnull final String functionName, + @Nonnull final List parameterTypes, + @Nonnull final EncapsulationFunction encapsulationFunction) { + super(functionName, parameterTypes, encapsulationFunction); + } + + protected BuiltInTableFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, + @Nullable final Type variadicSuffixType, @Nonnull final EncapsulationFunction encapsulationFunction) { + super(functionName, parameterTypes, variadicSuffixType, encapsulationFunction); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java index 90f7be1151..ab5a5ee58a 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java @@ -35,6 +35,7 @@ import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementInUnionRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementInsertRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementRecursiveUnionRule; +import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementTableFunctionRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementTempTableInsertRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementIntersectionRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementNestedLoopJoinRule; @@ -179,7 +180,8 @@ public class PlannerRuleSet { new ImplementInsertRule(), new ImplementTempTableInsertRule(), new ImplementUpdateRule(), - new ImplementRecursiveUnionRule() + new ImplementRecursiveUnionRule(), + new ImplementTableFunctionRule() ); private static final List> EXPLORATION_RULES = diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TableFunctionExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TableFunctionExpression.java new file mode 100644 index 0000000000..7cc842e14d --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TableFunctionExpression.java @@ -0,0 +1,167 @@ +/* + * TableFunctionExpression.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.expressions; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.ComparisonRange; +import com.apple.foundationdb.record.query.plan.cascades.Compensation; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.IdentityBiMap; +import com.apple.foundationdb.record.query.plan.cascades.MatchInfo; +import com.apple.foundationdb.record.query.plan.cascades.PartialMatch; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.explain.InternalPlannerGraphRewritable; +import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue; +import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.PullUp; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * A table function expression that delegates the actual execution to an underlying {@link StreamingValue} which + * effectively returns a stream of results. + */ +@API(API.Status.EXPERIMENTAL) +public class TableFunctionExpression implements RelationalExpression, InternalPlannerGraphRewritable { + @Nonnull + private final StreamingValue value; + + public TableFunctionExpression(@Nonnull final StreamingValue value) { + this.value = value; + } + + @Nonnull + @Override + public Value getResultValue() { + return new QueriedValue(value.getResultType()); + } + + @Nonnull + @Override + public Set getDynamicTypes() { + return value.getDynamicTypes(); + } + + @Nonnull + public StreamingValue getValue() { + return value; + } + + @Nonnull + @Override + public List getQuantifiers() { + return Collections.emptyList(); + } + + @Nonnull + @Override + public Set getCorrelatedTo() { + return value.getCorrelatedTo(); + } + + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public boolean equalsWithoutChildren(@Nonnull RelationalExpression otherExpression, + @Nonnull final AliasMap equivalencesMap) { + if (this == otherExpression) { + return true; + } + if (!(otherExpression instanceof TableFunctionExpression)) { + return false; + } + + final var otherTableFunctionExpression = (TableFunctionExpression)otherExpression; + + return value.semanticEquals(otherTableFunctionExpression.getValue(), equivalencesMap); + } + + @Override + public int hashCodeWithoutChildren() { + return Objects.hash(value); + } + + @Nonnull + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public TableFunctionExpression translateCorrelations(@Nonnull final TranslationMap translationMap, + final boolean shouldSimplifyValues, + @Nonnull final List translatedQuantifiers) { + final Value translatedCollectionValue = value.translateCorrelations(translationMap, shouldSimplifyValues); + if (translatedCollectionValue != value) { + return new TableFunctionExpression((StreamingValue)translatedCollectionValue); + } + return this; + } + + @Nonnull + @Override + public Iterable subsumedBy(@Nonnull final RelationalExpression candidateExpression, + @Nonnull final AliasMap bindingAliasMap, + @Nonnull final IdentityBiMap partialMatchMap, + @Nonnull final EvaluationContext evaluationContext) { + if (!isCompatiblyAndCompletelyBound(bindingAliasMap, candidateExpression.getQuantifiers())) { + return ImmutableList.of(); + } + + return exactlySubsumedBy(candidateExpression, bindingAliasMap, partialMatchMap, TranslationMap.empty()); + } + + @Nonnull + @Override + public Compensation compensate(@Nonnull final PartialMatch partialMatch, + @Nonnull final Map boundParameterPrefixMap, + @Nullable final PullUp pullUp, + @Nonnull final CorrelationIdentifier nestingAlias) { + // subsumedBy() is based on equality and this expression is always a leaf, thus we return empty here as + // if there is a match, it's exact + return Compensation.noCompensation(); + } + + @Nonnull + @Override + public PlannerGraph rewriteInternalPlannerGraph(@Nonnull final List childGraphs) { + return PlannerGraph.fromNodeAndChildGraphs( + new PlannerGraph.LogicalOperatorNode(this, + "TFunc", + ImmutableList.of(toString()), + ImmutableMap.of()), + childGraphs); + } + + @Override + public String toString() { + return value.toString(); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java index 0fe1401766..3c7b702c48 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java @@ -28,6 +28,7 @@ import com.apple.foundationdb.record.query.plan.cascades.expressions.GroupByExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.InsertExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.RecursiveUnionExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TableFunctionExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableInsertExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalDistinctExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalFilterExpression; @@ -246,6 +247,11 @@ public static BindingMatcher explodeExpression() { return ofTypeOwning(ExplodeExpression.class, CollectionMatcher.empty()); } + @Nonnull + public static BindingMatcher tableFunctionExpression() { + return ofTypeOwning(TableFunctionExpression.class, CollectionMatcher.empty()); + } + @Nonnull public static BindingMatcher groupByExpression(@Nonnull final BindingMatcher downstream) { return ofTypeOwning(GroupByExpression.class, only(downstream)); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/CompatibleTypeEvolutionPredicate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/CompatibleTypeEvolutionPredicate.java index 0399e57daf..8896be18d1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/CompatibleTypeEvolutionPredicate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/CompatibleTypeEvolutionPredicate.java @@ -37,6 +37,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; import com.apple.foundationdb.record.query.plan.cascades.BooleanWithConstraint; +import com.apple.foundationdb.record.query.plan.cascades.values.FirstOrDefaultStreamingValue; import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; import com.apple.foundationdb.record.query.plan.cascades.ValueEquivalence; @@ -290,7 +291,7 @@ private static List computeFieldAccessForDerivation( return resultTrieBuilders.build(); } - if (derivationValue instanceof FirstOrDefaultValue) { + if (derivationValue instanceof FirstOrDefaultValue || derivationValue instanceof FirstOrDefaultStreamingValue) { Verify.verify(nestedResults.size() == 2); return nestedResults.get(0); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java index c74fa0ede8..8c835f70d3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java @@ -36,6 +36,7 @@ import com.apple.foundationdb.record.query.plan.cascades.expressions.GroupByExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.InsertExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.RecursiveUnionExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TableFunctionExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableInsertExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalDistinctExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalFilterExpression; @@ -75,6 +76,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; @@ -248,6 +250,12 @@ public Cardinalities visitRecordQueryInsertPlan(@Nonnull final RecordQueryInsert return fromChild(insertPlan); } + @Nonnull + @Override + public Cardinalities visitRecordQueryTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) { + return Cardinalities.unknownMaxCardinality(); + } + @Nonnull @Override public Cardinalities visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { @@ -559,6 +567,12 @@ public Cardinalities visitLogicalIntersectionExpression(@Nonnull final LogicalIn return intersectCardinalities(fromChildren(logicalIntersectionExpression)); } + @Nonnull + @Override + public Cardinalities visitTableFunctionExpression(@Nonnull final TableFunctionExpression element) { + return Cardinalities.unknownMaxCardinality(); + } + @Nonnull @Override public Cardinalities visitLogicalUniqueExpression(@Nonnull final LogicalUniqueExpression logicalUniqueExpression) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java index b2fe773d8c..92b7493ea2 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java @@ -29,6 +29,7 @@ import com.apple.foundationdb.record.query.plan.cascades.Reference; import com.apple.foundationdb.record.query.plan.cascades.PlanProperty; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.values.FirstOrDefaultStreamingValue; import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; import com.apple.foundationdb.record.query.plan.cascades.TreeLike; import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; @@ -60,6 +61,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; @@ -336,6 +338,15 @@ public Derivations visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPl return new Derivations(resultValuesBuilder.build(), localValuesBuilder.build()); } + @Nonnull + @Override + public Derivations visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan tableFunctionPlan) { + final var streamingValue = tableFunctionPlan.getValue(); + final var elementType = streamingValue.getResultType(); + final var values = ImmutableList.of(new FirstOrDefaultStreamingValue(streamingValue, new ThrowsValue(elementType))); + return new Derivations(values, values); + } + @Nonnull @Override public Derivations visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java index ecb0a4a020..3a62e651cb 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java @@ -47,6 +47,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; @@ -214,6 +215,12 @@ public Boolean visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPlan) return distinctRecordsFromSingleChild(insertPlan); } + @Nonnull + @Override + public Boolean visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) { + return false; + } + @Nonnull @Override public Boolean visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java index e994a83b62..3e82785a1e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java @@ -59,6 +59,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; @@ -302,6 +303,12 @@ public Ordering visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPlan) return Ordering.empty(); } + @Nonnull + @Override + public Ordering visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) { + return Ordering.empty(); + } + @Nonnull @Override public Ordering visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java index f984611bb2..cd714cfb76 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java @@ -48,6 +48,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; @@ -216,6 +217,12 @@ public Optional> visitInsertPlan(@Nonnull final RecordQueryInsertPla return Optional.empty(); } + @Nonnull + @Override + public Optional> visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) { + return Optional.empty(); + } + @Nonnull @Override public Optional> visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java index ab2584f85a..8a82cd5b34 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java @@ -45,6 +45,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; @@ -203,6 +204,12 @@ public Boolean visitInsertPlan(@Nonnull final RecordQueryInsertPlan element) { return true; } + @Nonnull + @Override + public Boolean visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) { + return false; + } + @Nonnull @Override public Boolean visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan element) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTableFunctionRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTableFunctionRule.java new file mode 100644 index 0000000000..d9d88930b4 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTableFunctionRule.java @@ -0,0 +1,51 @@ +/* + * ImplementTableFunctionRule.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.rules; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.query.plan.cascades.CascadesRule; +import com.apple.foundationdb.record.query.plan.cascades.CascadesRuleCall; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TableFunctionExpression; +import com.apple.foundationdb.record.query.plan.cascades.matching.structure.BindingMatcher; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; + +import javax.annotation.Nonnull; + +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RelationalExpressionMatchers.tableFunctionExpression; + +/** + * A rule that implements a table function expression into a {@link RecordQueryTableFunctionPlan}. + */ +@API(API.Status.EXPERIMENTAL) +public class ImplementTableFunctionRule extends CascadesRule { + private static final BindingMatcher root = + tableFunctionExpression(); + + public ImplementTableFunctionRule() { + super(root); + } + + @Override + public void onMatch(@Nonnull final CascadesRuleCall call) { + final var tableFunctionExpression = call.get(root); + call.yieldExpression(new RecordQueryTableFunctionPlan(tableFunctionExpression.getValue())); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java index 1044904c62..f7b57e3850 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java @@ -2801,6 +2801,24 @@ public Type getElementType() { return elementType; } + /** + * Return the array with a given element {@link Type} and the same nullability semantics. + * @param elementType The new element type, can be {@code null}. + * @return the array with a given element {@link Type} and the same nullability semantics, if the element type + * matches the current element type, the same instance is returned. + */ + @Nonnull + @SuppressWarnings("PMD.BrokenNullCheck") // I think PMD got confused or the null check conjunctions below. + public Type.Array withElementType(@Nullable final Type elementType) { + if (elementType == null && this.elementType == null) { + return this; + } + if (elementType != null && elementType.equals(this.elementType)) { + return this; + } + return new Array(isNullable(), elementType); + } + /** * Checks whether the array type is erased or not. * diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FirstOrDefaultStreamingValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FirstOrDefaultStreamingValue.java new file mode 100644 index 0000000000..e1ece731e4 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FirstOrDefaultStreamingValue.java @@ -0,0 +1,187 @@ +/* + * FirstOrDefaultStreamingValue.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.planprotos.PFirstOrDefaultStreamingValue; +import com.apple.foundationdb.record.planprotos.PFirstOrDefaultValue; +import com.apple.foundationdb.record.planprotos.PValue; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; +import com.google.auto.service.AutoService; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; + +/** + * A value that returns the first element of the streaming {@link Value} that is passed in. If the streaming value returns + * and empty stream or the first item is null, a default value of the same type is returned. + * + * @see RecordCursor#first() semantics for more information. + */ +@API(API.Status.EXPERIMENTAL) +public class FirstOrDefaultStreamingValue extends AbstractValue { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("First-Or-Default-Streaming-Value"); + + @Nonnull + private final StreamingValue childValue; + @Nonnull + private final Value onEmptyResultValue; + @Nonnull + private final Supplier> childrenSupplier; + @Nonnull + private final Type resultType; + + public FirstOrDefaultStreamingValue(@Nonnull final StreamingValue childValue, @Nonnull final Value onEmptyResultValue) { + this.childValue = childValue; + this.onEmptyResultValue = onEmptyResultValue; + this.childrenSupplier = () -> ImmutableList.of(childValue, onEmptyResultValue); + this.resultType = Objects.requireNonNull(childValue.getResultType()); + } + + @Nonnull + @Override + public List computeChildren() { + return childrenSupplier.get(); + } + + @Nonnull + @Override + public Value withChildren(final Iterable newChildren) { + final var newChildrenList = ImmutableList.copyOf(newChildren); + Verify.verify(newChildrenList.size() == 2); + Verify.verify(newChildrenList.get(0) instanceof StreamingValue); + return new FirstOrDefaultStreamingValue((StreamingValue)newChildrenList.get(0), newChildrenList.get(1)); + } + + @Nonnull + @Override + public Type getResultType() { + return resultType; + } + + @Nonnull + public Value getOnEmptyResultValue() { + return onEmptyResultValue; + } + + @Override + @SpotBugsSuppressWarnings({"NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE", "NP_NONNULL_PARAM_VIOLATION"}) + public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { + final var childResult = childValue.evalAsStream(store, context, null, null).first().join(); + if (childResult.isPresent()) { + return childResult.get(); + } else { + return onEmptyResultValue.eval(store, context); + } + } + + @Override + public int hashCodeWithoutChildren() { + return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH); + } + + @Override + public int planHash(@Nonnull final PlanHashMode mode) { + return PlanHashable.objectsPlanHash(mode, BASE_HASH, childValue, onEmptyResultValue); + } + + @Nonnull + @Override + public ExplainTokensWithPrecedence explain(@Nonnull final Iterable> explainSuppliers) { + return ExplainTokensWithPrecedence.of(new ExplainTokens().addFunctionCall("firstOrDefault", + Value.explainFunctionArguments(explainSuppliers))); + } + + @Override + public int hashCode() { + return semanticHashCode(); + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @SpotBugsSuppressWarnings("EQ_UNUSUAL") + @Override + public boolean equals(final Object other) { + return semanticEquals(other, AliasMap.emptyMap()); + } + + @Nonnull + @Override + public PFirstOrDefaultValue toProto(@Nonnull final PlanSerializationContext serializationContext) { + return PFirstOrDefaultValue.newBuilder() + .setChildValue(childValue.toValueProto(serializationContext)) + .setOnEmptyResultValue(onEmptyResultValue.toValueProto(serializationContext)) + .build(); + } + + @Nonnull + @Override + public PValue toValueProto(@Nonnull PlanSerializationContext serializationContext) { + return PValue.newBuilder().setFirstOrDefaultValue(toProto(serializationContext)).build(); + } + + @Nonnull + public static FirstOrDefaultStreamingValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PFirstOrDefaultStreamingValue firstOrDefaultStreamingValueProto) { + final var value = Value.fromValueProto(serializationContext, Objects.requireNonNull(firstOrDefaultStreamingValueProto.getChildValue())); + if (!(value instanceof StreamingValue)) { + throw new RecordCoreException("invalid value, expecting streaming value").addLogInfo(LogMessageKeys.VALUE, value); + } + return new FirstOrDefaultStreamingValue((StreamingValue)value, + Value.fromValueProto(serializationContext, Objects.requireNonNull(firstOrDefaultStreamingValueProto.getOnEmptyResultValue()))); + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PFirstOrDefaultStreamingValue.class; + } + + @Nonnull + @Override + public FirstOrDefaultStreamingValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PFirstOrDefaultStreamingValue firstOrDefaultValueStreamingProto) { + return FirstOrDefaultStreamingValue.fromProto(serializationContext, firstOrDefaultValueStreamingProto); + } + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java new file mode 100644 index 0000000000..c7e4718759 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java @@ -0,0 +1,416 @@ +/* + * RangeValue.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.RecordCursorContinuation; +import com.apple.foundationdb.record.RecordCursorProto; +import com.apple.foundationdb.record.RecordCursorResult; +import com.apple.foundationdb.record.RecordCursorVisitor; +import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.planprotos.PRangeValue; +import com.apple.foundationdb.record.planprotos.PValue; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; +import com.apple.foundationdb.record.query.plan.cascades.BuiltInTableFunction; +import com.apple.foundationdb.record.query.plan.cascades.Column; +import com.apple.foundationdb.record.query.plan.cascades.SemanticException; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; +import com.apple.foundationdb.record.query.plan.plans.QueryResult; +import com.apple.foundationdb.tuple.ByteArrayUtil2; +import com.google.auto.service.AutoService; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * A {@link StreamingValue} that is able to return a range that is defined as the following: + *
    + *
  • an optional inclusive start (0L by default).
    • + *
  • an exclusive end.
    • + *
  • an optional step (1L by default).
    • + *
+ * For more information, see relational SQL {@code range} table-valued function. + */ +public class RangeValue extends AbstractValue implements StreamingValue, CreatesDynamicTypesValue { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Range-Value"); + + @Nonnull + private final Value endExclusive; + + @Nonnull + private final Value beginInclusive; + + @Nonnull + private final Value step; + + @Nonnull + private final Value currentRangeValue; + + public RangeValue(@Nonnull final Value endExclusive, @Nonnull final Value beginInclusive, + @Nonnull final Value step) { + this.endExclusive = endExclusive; + this.beginInclusive = beginInclusive; + this.step = step; + currentRangeValue = RecordConstructorValue.ofColumns(ImmutableList.of(Column.of(Optional.of("ID"), LiteralValue.ofScalar(-1L)))); + } + + @Nonnull + @Override + public Type.Record getResultType() { + return (Type.Record)currentRangeValue.getResultType(); + } + + @Nonnull + @Override + public RecordCursor evalAsStream(@Nonnull final FDBRecordStoreBase store, + @Nonnull final EvaluationContext context, + @Nullable final byte[] continuation, + @Nonnull final ExecuteProperties executeProperties) { + final long endExclusiveValue = (Long)Verify.verifyNotNull(endExclusive.eval(store, context)); + final var beginInclusiveValue = (Long)Verify.verifyNotNull(this.beginInclusive.eval(store, context)); + final var stepValue = (Long)Verify.verifyNotNull(step.eval(store, context)); + return new Cursor(store.getExecutor(), endExclusiveValue, beginInclusiveValue, stepValue, rangeValueAsLong -> Objects.requireNonNull(currentRangeValue.replace(v -> { + if (v instanceof LiteralValue) { + return LiteralValue.ofScalar(rangeValueAsLong); + } + return v; + })).eval(store, context), continuation).skipThenLimit(executeProperties.getSkip(), executeProperties.getReturnedRowLimit()); + } + + @Nonnull + @Override + public ExplainTokensWithPrecedence explain(@Nonnull final Iterable> explainSuppliers) { + final var endExplainTokens = Iterables.get(explainSuppliers, 0).get().getExplainTokens(); + final var beginExplainTokens = Iterables.get(explainSuppliers, 1).get().getExplainTokens(); + final var stepExplainTokens = Iterables.get(explainSuppliers, 2).get().getExplainTokens(); + + return ExplainTokensWithPrecedence.of(new ExplainTokens().addFunctionCall("range", + new ExplainTokens().addSequence(() -> new ExplainTokens().addCommaAndWhiteSpace(), + beginExplainTokens, + endExplainTokens, + new ExplainTokens().addKeyword("STEP").addWhitespace().addNested(stepExplainTokens)))); + } + + @Nullable + @Override + public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { + throw new IllegalStateException("unable to eval an streaming value with eval()"); + } + + @Override + public int hashCodeWithoutChildren() { + return PlanHashable.objectPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH); + } + + @Nonnull + @Override + public PValue toValueProto(@Nonnull final PlanSerializationContext serializationContext) { + return PValue.newBuilder().setRangeValue(toProto(serializationContext)).build(); + } + + @Override + public int planHash(@Nonnull final PlanHashMode hashMode) { + return PlanHashable.objectsPlanHash(hashMode, BASE_HASH, getChildren()); + } + + @Nonnull + @Override + public PRangeValue toProto(@Nonnull final PlanSerializationContext serializationContext) { + final PRangeValue.Builder builder = PRangeValue.newBuilder(); + builder.setEndExclusiveChild(endExclusive.toValueProto(serializationContext)); + builder.setBeginInclusiveChild(beginInclusive.toValueProto(serializationContext)); + builder.setStepChild(step.toValueProto(serializationContext)); + return builder.build(); + } + + @Nonnull + @Override + protected Iterable computeChildren() { + final ImmutableList.Builder childrenBuilder = ImmutableList.builder(); + childrenBuilder.add(endExclusive); + childrenBuilder.add(beginInclusive); + childrenBuilder.add(step); + return childrenBuilder.build(); + } + + @Nonnull + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") // intentional. + public Value withChildren(final Iterable newChildren) { + final var newChildrenSize = Iterables.size(newChildren); + Verify.verify(newChildrenSize == 3); + final var newEndExclusive = Iterables.get(newChildren, 0); + final var newBeginInclusive = Iterables.get(newChildren, 1); + final var newStep = Iterables.get(newChildren, 2); + + if (newEndExclusive == endExclusive && newBeginInclusive == beginInclusive && newStep == step) { + return this; + } + + return new RangeValue(newEndExclusive, newBeginInclusive, newStep); + } + + @Nonnull + public static RangeValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PRangeValue rangeValueProto) { + final var endExclusive = Value.fromValueProto(serializationContext, rangeValueProto.getEndExclusiveChild()); + final var beginInclusive = Value.fromValueProto(serializationContext, rangeValueProto.getBeginInclusiveChild()); + final var step = Value.fromValueProto(serializationContext, rangeValueProto.getStepChild()); + return new RangeValue(endExclusive, beginInclusive, step); + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PRangeValue.class; + } + + @Nonnull + @Override + public RangeValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PRangeValue rangeValueProto) { + return RangeValue.fromProto(serializationContext, rangeValueProto); + } + } + + public static class Cursor implements RecordCursor { + + @Nonnull + private final Executor executor; + + private final long endExclusive; + + private final long step; + + private long nextPosition; // position of the next value to return + + private boolean closed = false; + + @Nonnull + private final Function rangeValueCreator; + + Cursor(@Nonnull final Executor executor, final long endExclusive, final long beginInclusive, + final long step, @Nonnull final Function rangeValueCreator, @Nullable final byte[] continuation) { + this(executor, endExclusive, step, rangeValueCreator, + continuation == null ? beginInclusive : Continuation.from(continuation, endExclusive, step).getNextPosition()); + } + + private Cursor(@Nonnull final Executor executor, final long endExclusive, final long step, + @Nonnull final Function rangeValueCreator, final long nextPosition) { + checkValidRange(nextPosition, endExclusive, step); + this.executor = executor; + this.endExclusive = endExclusive; + this.step = step; + this.nextPosition = nextPosition; + this.rangeValueCreator = rangeValueCreator; + } + + @Nonnull + @Override + public CompletableFuture> onNext() { + return CompletableFuture.completedFuture(getNext()); + } + + @Nonnull + @Override + public RecordCursorResult getNext() { + RecordCursorResult nextResult; + if (nextPosition < endExclusive) { + final var continuation = new Continuation(endExclusive, nextPosition + step, step); + nextResult = RecordCursorResult.withNextValue(QueryResult.ofComputed(rangeValueCreator.apply(nextPosition)), continuation); + nextPosition += step; + } else { + nextResult = RecordCursorResult.exhausted(); + } + return nextResult; + } + + @Override + public void close() { + closed = true; + } + + @Override + public boolean isClosed() { + return closed; + } + + @Override + public boolean accept(@Nonnull RecordCursorVisitor visitor) { + visitor.visitEnter(this); + return visitor.visitLeave(this); + } + + @Override + @Nonnull + public Executor getExecutor() { + return executor; + } + + private static void checkValidRange(final long position, final long endExclusive, final long step) { + if (position < 0L) { + throw new RecordCoreException("only non-negative position is allowed in range"); + } + if (endExclusive < 0L) { + throw new RecordCoreException("only non-negative exclusive end is allowed in range"); + } + if (step <= 0L) { + throw new RecordCoreException("only positive step is allowed in range"); + } + } + + private static class Continuation implements RecordCursorContinuation { + private final long endExclusive; + private final long nextPosition; + private final long step; + + public Continuation(final long endExclusive, final long nextPosition, final long step) { + this.nextPosition = nextPosition; + this.endExclusive = endExclusive; + this.step = step; + } + + @Override + public boolean isEnd() { + // If a next value is returned as part of a cursor result, the continuation must not be an end continuation + // (i.e., isEnd() must be false), per the contract of RecordCursorResult. This is the case even if the + // cursor knows for certain that there is no more after that result, as in the ListCursor. + return nextPosition >= endExclusive + step; + } + + public long getNextPosition() { + return nextPosition; + } + + @Nonnull + @Override + public ByteString toByteString() { + if (isEnd()) { + return ByteString.EMPTY; + } + final var protoBuilder = RecordCursorProto.RangeCursorContinuation.newBuilder().setNextPosition(nextPosition); + return protoBuilder.build().toByteString(); + } + + @Nullable + @Override + public byte[] toBytes() { + return toByteString().toByteArray(); + } + + @Nonnull + public static Continuation from(@Nonnull final RecordCursorProto.RangeCursorContinuation message, + final long endExclusive, final long step) { + final var nextPosition = message.getNextPosition(); + return new Continuation(endExclusive, nextPosition, step); + } + + @Nonnull + public static Continuation from(@Nonnull final byte[] unparsedContinuationBytes, final long endExclusive, + final long step) { + try { + final var parsed = RecordCursorProto.RangeCursorContinuation.parseFrom(unparsedContinuationBytes); + return from(parsed, endExclusive, step); + } catch (InvalidProtocolBufferException ex) { + throw new RecordCoreException("invalid continuation", ex) + .addLogInfo(LogMessageKeys.RAW_BYTES, ByteArrayUtil2.loggable(unparsedContinuationBytes)); + } + } + } + } + + /** + * The {@code range} table function. + */ + @AutoService(BuiltInFunction.class) + public static class RangeFn extends BuiltInTableFunction { + + @Nonnull + private static StreamingValue encapsulateInternal(@Nonnull final List arguments) { + Verify.verify(!arguments.isEmpty()); + final Value endExclusive; + final Value beginInclusiveMaybe; + final Value stepMaybe; + if (arguments.size() == 1) { + endExclusive = PromoteValue.inject((Value)arguments.get(0), Type.primitiveType(Type.TypeCode.LONG)); + checkValidBoundaryType(endExclusive); + beginInclusiveMaybe = LiteralValue.ofScalar(0L); + stepMaybe = LiteralValue.ofScalar(1L); + } else { + checkValidBoundaryType((Value)arguments.get(0)); + beginInclusiveMaybe = PromoteValue.inject((Value)arguments.get(0), Type.primitiveType(Type.TypeCode.LONG)); + checkValidBoundaryType((Value)arguments.get(1)); + endExclusive = PromoteValue.inject((Value)arguments.get(1), Type.primitiveType(Type.TypeCode.LONG)); + if (arguments.size() > 2) { + checkValidBoundaryType((Value)arguments.get(2)); + stepMaybe = PromoteValue.inject((Value)arguments.get(2), Type.primitiveType(Type.TypeCode.LONG)); + } else { + stepMaybe = LiteralValue.ofScalar(1L); + } + } + return new RangeValue(endExclusive, beginInclusiveMaybe, stepMaybe); + } + + public RangeFn() { + super("range", ImmutableList.of(), new Type.Any(), (ignored, arguments) -> encapsulateInternal(arguments)); + } + } + + private static void checkValidBoundaryType(@Nonnull final Value value) { + final var type = value.getResultType(); + SemanticException.check(type.isPrimitive(), SemanticException.ErrorCode.INCOMPATIBLE_TYPE); + final var maxType = Type.maximumType(type, Type.primitiveType(Type.TypeCode.LONG)); + SemanticException.check(maxType != null, SemanticException.ErrorCode.INCOMPATIBLE_TYPE); + // currently, we do not support non-deterministic boundaries since we do not pre-compute them and store + // them in the continuation, but we can change enable this if required. + SemanticException.check(value.preOrderStream().filter(NondeterministicValue.class::isInstance).findAny().isEmpty(), + SemanticException.ErrorCode.UNSUPPORTED); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java new file mode 100644 index 0000000000..c70ad58e20 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java @@ -0,0 +1,52 @@ +/* + * StreamValue.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.plans.QueryResult; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** + * a {@link Value} that returns a stream of results upon evaluation. + */ +public interface StreamingValue extends Value { + + /** + * Returns a {@link RecordCursor} over the result stream returned upon evaluation. + * @param store The record store used to fetch and evaluate the results. + * @param context The evaluation context, containing a set of contextual parameters for evaluating the results. + * @param continuation The continuation bytes used to resume the evaluation of the result stream. + * @param executeProperties The execution properties. + * @param The type of the returned results when fetched from a record store. + * @return A cursor over the result stream returned upon evaluation. + */ + @Nonnull + RecordCursor evalAsStream(@Nonnull FDBRecordStoreBase store, + @Nonnull EvaluationContext context, + @Nullable byte[] continuation, + @Nonnull ExecuteProperties executeProperties); +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java index 05de1220e5..d2ec7c1066 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java @@ -62,6 +62,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTextIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTypeFilterPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryUnionOnKeyExpressionPlan; @@ -414,6 +415,13 @@ public ExplainTokens visitInsertPlan(@Nonnull final RecordQueryInsertPlan insert .addIdentifier(insertPlan.getTargetRecordType()); } + @Nonnull + @Override + public ExplainTokens visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan tableFunctionPlan) { + return addKeyword("TF").addWhitespace() + .addNested(tableFunctionPlan.getValue().explain().getExplainTokens()); + } + @Nonnull @Override public ExplainTokens visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java new file mode 100644 index 0000000000..45e8d323d7 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java @@ -0,0 +1,279 @@ +/* + * RecordQueryTableFunctionPlan.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.plans; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.planprotos.PRecordQueryPlan; +import com.apple.foundationdb.record.planprotos.PRecordQueryTableFunctionPlan; +import com.apple.foundationdb.record.provider.common.StoreTimer; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.AvailableFields; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.Memoizer; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.explain.Attribute; +import com.apple.foundationdb.record.query.plan.cascades.explain.NodeInfo; +import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph; +import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue; +import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.apple.foundationdb.record.query.plan.explain.ExplainPlanVisitor; +import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * A query plan that delegates its execution to a table-valued {@link StreamingValue}. + */ +@API(API.Status.INTERNAL) +public class RecordQueryTableFunctionPlan implements RecordQueryPlanWithNoChildren { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Table-Function-Plan"); + + @Nonnull + private final StreamingValue value; + + public RecordQueryTableFunctionPlan(@Nonnull final StreamingValue collectionValue) { + this.value = collectionValue; + } + + @Nonnull + @Override + public RecordCursor executePlan(@Nonnull final FDBRecordStoreBase store, + @Nonnull final EvaluationContext context, + @Nullable final byte[] continuation, + @Nonnull final ExecuteProperties executeProperties) { + return value.evalAsStream(store, context, continuation, executeProperties); + } + + @Nonnull + @Override + public Set getCorrelatedTo() { + return value.getCorrelatedTo(); + } + + @Nonnull + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public RecordQueryTableFunctionPlan translateCorrelations(@Nonnull final TranslationMap translationMap, + final boolean shouldSimplifyValues, + @Nonnull final List translatedQuantifiers) { + final Value translatedValue = value.translateCorrelations(translationMap, shouldSimplifyValues); + if (translatedValue != value) { + return new RecordQueryTableFunctionPlan((StreamingValue)translatedValue); + } + return this; + } + + @Override + public boolean isReverse() { + return false; + } + + @Override + public RecordQueryTableFunctionPlan strictlySorted(@Nonnull final Memoizer memoizer) { + return this; + } + + @Override + public boolean hasRecordScan() { + return false; + } + + @Override + public boolean hasFullRecordScan() { + return false; + } + + @Override + public boolean hasIndexScan(@Nonnull final String indexName) { + return false; + } + + @Nonnull + @Override + public Set getUsedIndexes() { + return ImmutableSet.of(); + } + + @Override + public boolean hasLoadBykeys() { + return false; + } + + @Nonnull + @Override + public AvailableFields getAvailableFields() { + return AvailableFields.NO_FIELDS; + } + + @Nonnull + @Override + public Value getResultValue() { + return new QueriedValue(value.getResultType()); + } + + @Nonnull + public StreamingValue getValue() { + return value; + } + + @Nonnull + @Override + public Set getDynamicTypes() { + return value.getDynamicTypes(); + } + + + @Nonnull + @Override + public String toString() { + return ExplainPlanVisitor.toStringForDebugging(this); + } + + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public boolean equalsWithoutChildren(@Nonnull final RelationalExpression otherExpression, + @Nonnull final AliasMap equivalencesMap) { + if (this == otherExpression) { + return true; + } + if (getClass() != otherExpression.getClass()) { + return false; + } + final var otherTableFunctionPlan = (RecordQueryTableFunctionPlan)otherExpression; + + return value.semanticEquals(otherTableFunctionPlan.value, equivalencesMap); + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @Override + public boolean equals(final Object other) { + return structuralEquals(other); + } + + @Override + public int hashCode() { + return structuralHashCode(); + } + + @Override + public int hashCodeWithoutChildren() { + return Objects.hash(getResultValue()); + } + + @Override + public void logPlanStructure(StoreTimer timer) { + // nothing to increment + } + + @Override + public int getComplexity() { + return 1; + } + + @Override + public int planHash(@Nonnull final PlanHashMode mode) { + switch (mode.getKind()) { + case LEGACY: + case FOR_CONTINUATION: + return PlanHashable.objectsPlanHash(mode, BASE_HASH, getResultValue()); + default: + throw new UnsupportedOperationException("Hash kind " + mode.getKind() + " is not supported"); + } + } + + @Nonnull + @Override + public PlannerGraph rewritePlannerGraph(@Nonnull final List childGraphs) { + return PlannerGraph.fromNodeAndChildGraphs( + new PlannerGraph.OperatorNodeWithInfo(this, + NodeInfo.VALUE_COMPUTATION_OPERATOR, + ImmutableList.of("TFUNC {{expr}}"), + ImmutableMap.of("expr", Attribute.gml(value.toString()))), + childGraphs); + } + + @Nonnull + @Override + public PRecordQueryTableFunctionPlan toProto(@Nonnull final PlanSerializationContext serializationContext) { + return PRecordQueryTableFunctionPlan.newBuilder() + .setValue(value.toValueProto(serializationContext)) + .build(); + } + + @Nonnull + @Override + public PRecordQueryPlan toRecordQueryPlanProto(@Nonnull final PlanSerializationContext serializationContext) { + return PRecordQueryPlan.newBuilder().setTableFunctionPlan(toProto(serializationContext)).build(); + } + + @Nonnull + public static RecordQueryTableFunctionPlan fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PRecordQueryTableFunctionPlan recordQueryTableFunctionPlanProto) { + final var value = Value.fromValueProto(serializationContext, + Objects.requireNonNull(recordQueryTableFunctionPlanProto.getValue())); + if (!(value instanceof StreamingValue)) { + throw new RecordCoreException("invalid value, expecting streaming value").addLogInfo(LogMessageKeys.VALUE, value); + } + return new RecordQueryTableFunctionPlan((StreamingValue)value); + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PRecordQueryTableFunctionPlan.class; + } + + @Nonnull + @Override + public RecordQueryTableFunctionPlan fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PRecordQueryTableFunctionPlan message) { + return RecordQueryTableFunctionPlan.fromProto(serializationContext, message); + } + } +} + diff --git a/fdb-record-layer-core/src/main/proto/record_cursor.proto b/fdb-record-layer-core/src/main/proto/record_cursor.proto index c1400c3c8a..5c93228902 100644 --- a/fdb-record-layer-core/src/main/proto/record_cursor.proto +++ b/fdb-record-layer-core/src/main/proto/record_cursor.proto @@ -131,4 +131,8 @@ message RecursiveCursorContinuation { optional bool isInitialState = 1; optional planprotos.PTempTable tempTable = 2; optional bytes activeStateContinuation = 3; +} + +message RangeCursorContinuation { + optional int64 nextPosition = 1; } \ No newline at end of file diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 1ed5d7c474..33bcf47d8c 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -251,6 +251,8 @@ message PValue { PNumericAggregationValue.PBitmapConstructAgg numeric_aggregation_value_bitmap_construct_agg = 45; PQuantifiedRecordValue quantified_record_value = 46; PMacroFunctionValue macro_function_value = 47; + PRangeValue range_value = 48; + PFirstOrDefaultStreamingValue first_or_default_streaming_value = 49; } } @@ -460,6 +462,11 @@ message PFirstOrDefaultValue { optional PValue on_empty_result_value = 2; } +message PFirstOrDefaultStreamingValue { + optional PValue child_value = 1; + optional PValue on_empty_result_value = 2; +} + message PFromOrderedBytesValue { optional PValue child = 1; optional PDirection direction = 2; @@ -1005,6 +1012,12 @@ message PCollateValue { optional PValue strength_child = 4; } +message PRangeValue { + optional PValue end_exclusive_child = 1; + optional PValue begin_inclusive_child = 2; + optional PValue step_child = 3; +} + // // Comparisons // @@ -1260,6 +1273,7 @@ message PRecordQueryPlan { PTempTableScanPlan temp_table_scan_plan = 34; PTempTableInsertPlan temp_table_insert_plan = 35; PRecursiveUnionQueryPlan recursive_union_query_plan = 36; + PRecordQueryTableFunctionPlan table_function_plan = 37; } } @@ -1852,3 +1866,10 @@ message PRecursiveUnionQueryPlan { optional string initialTempTableAlias = 3; optional string recursiveTempTableAlias = 4; } + +// +// PRecordQueryTableFunctionPlan +// +message PRecordQueryTableFunctionPlan { + optional PValue value = 1; +} diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/exceptions/ErrorCode.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/exceptions/ErrorCode.java index a145bbc2a9..29694190ce 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/exceptions/ErrorCode.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/exceptions/ErrorCode.java @@ -58,7 +58,7 @@ * are in-line with published SQL codes whenever possible). However, often you'll find that the standard does * not have an error code that matches what you're trying to do. In that case, choose a class from * the above table, and then define a unique code. - * Newly introduced error codes follow the pattern "..V..". For example: 08F01 + * Newly introduced error codes follow the pattern "..F..". For example: 08F01 * */ public enum ErrorCode { // Class 00 - Successful Completion @@ -122,6 +122,7 @@ public enum ErrorCode { INVALID_TABLE_DEFINITION("42F16"), UNKNOWN_TYPE("42F18"), INVALID_RECURSION("42F19"), + INCOMPATIBLE_TABLE_ALIAS("42F20"), /** * Indicates that a schema with the given name is already mapped to a schema template. */ diff --git a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 index 72879bb762..b7892eb08a 100644 --- a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 @@ -166,7 +166,6 @@ PARTITION: 'PARTITION'; PRIMARY: 'PRIMARY'; PROCEDURE: 'PROCEDURE'; PURGE: 'PURGE'; -RANGE: 'RANGE'; READ: 'READ'; READS: 'READS'; RECURSIVE: 'RECURSIVE'; diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index af53567851..6784e2c33d 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -226,6 +226,14 @@ namedQuery : name=fullId (columnAliases=fullIdList)? AS? '(' query ')' ; +tableFunction + : tableFunctionName '(' functionArgs? ')' inlineTableDefinition? + ; + +tableFunctionName + : fullId + ; + continuation : WITH CONTINUATION continuationAtom ; @@ -282,8 +290,10 @@ tableSource // done ; tableSourceItem // done - : tableName (AS? alias=uid)? (indexHint (',' indexHint)* )? #atomTableItem // done - | query AS? alias=uid #subqueryTableItem // done + : tableName (AS? alias=uid)? (indexHint (',' indexHint)* )? #atomTableItem // done + | '(' query ')' AS? alias=uid #subqueryTableItem // done + | VALUES recordConstructorForInlineTable (',' recordConstructorForInlineTable )* inlineTableDefinition? #inlineTableItem + | tableFunction (AS? alias=uid)? #tableValuedFunction ; indexHint @@ -296,6 +306,10 @@ indexHintType : JOIN | ORDER BY | GROUP BY ; +inlineTableDefinition + : AS? tableName uidListWithNestingsInParens + ; + joinPart : (INNER | CROSS)? JOIN tableSourceItem ( @@ -730,11 +744,15 @@ expressionsWithDefaults ; recordConstructorForInsert - : LEFT_ROUND_BRACKET expressionWithOptionalName (',' expressionWithOptionalName)* RIGHT_ROUND_BRACKET + : '(' expressionWithOptionalName (',' expressionWithOptionalName)* ')' + ; + +recordConstructorForInlineTable + : '(' expressionWithOptionalName (',' expressionWithOptionalName)* ')' ; recordConstructor - : ofTypeClause? LEFT_ROUND_BRACKET (uid DOT STAR | STAR | expressionWithName | expressionWithOptionalName (',' expressionWithOptionalName)*) RIGHT_ROUND_BRACKET + : ofTypeClause? '(' (uid DOT STAR | STAR | expressionWithName /* this can be removed */ | expressionWithOptionalName (',' expressionWithOptionalName)*) ')' ; ofTypeClause @@ -1132,7 +1150,7 @@ keywordsCanBeId | TEXT | TEMPORARY | TEMPTABLE | THAN | TRADITIONAL | TRANSACTION | TRANSACTIONAL | TRIGGERS | TRUNCATE | UNDEFINED | UNDOFILE | UNDO_BUFFER_SIZE | UNINSTALL | UNKNOWN | UNTIL | UPGRADE | USA | USER | USE_FRM | USER_RESOURCES - | VALIDATION | VALUE | VALUES | VAR_POP | VAR_SAMP | VARIABLES | VARIANCE | VERSION_TOKEN_ADMIN | VIEW | WAIT | WARNINGS | WITHOUT + | VALIDATION | VALUE | VAR_POP | VAR_SAMP | VARIABLES | VARIANCE | VERSION_TOKEN_ADMIN | VIEW | WAIT | WARNINGS | WITHOUT | WRAPPER | X509 | XA | XA_RECOVER_ADMIN | XML ; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java index beb60dcf77..bd170172d6 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java @@ -266,7 +266,7 @@ private static LogicalOperator generateCorrelatedFieldAccess(@Nonnull Expression } @Nonnull - private static Expressions convertToExpressions(@Nonnull Quantifier quantifier) { + public static Expressions convertToExpressions(@Nonnull Quantifier quantifier) { final ImmutableList.Builder attributesBuilder = ImmutableList.builder(); int colCount = 0; final var columns = quantifier.getFlowedColumns(); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index d41445fc29..2038a99ab7 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -24,6 +24,7 @@ import com.apple.foundationdb.record.query.plan.cascades.AccessHint; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.BuiltInTableFunction; import com.apple.foundationdb.record.query.plan.cascades.Correlated; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.IndexAccessHint; @@ -735,15 +736,21 @@ public static void validateContinuation(@Nonnull Expression continuation) { * @param functionName The function name. * @param flattenSingleItemRecords {@code true} if single-item records should be (recursively) replaced with their * content, otherwise {@code false}. + * @param isTableValued {@code true} if the function is expected to be a table-valued function, otherwise {@code false}. * @param arguments The function arguments. * @return A resolved SQL function {@code Expression}. */ @Nonnull - public Expression resolveFunction(@Nonnull final String functionName, boolean flattenSingleItemRecords, + public Expression resolveFunction(@Nonnull final String functionName, + boolean flattenSingleItemRecords, + boolean isTableValued, @Nonnull final Expression... arguments) { Assert.thatUnchecked(functionCatalog.containsFunction(functionName), ErrorCode.UNSUPPORTED_QUERY, () -> String.format(Locale.ROOT, "Unsupported operator %s", functionName)); final var builtInFunction = functionCatalog.lookUpFunction(functionName, arguments); + if (isTableValued) { + Assert.thatUnchecked(builtInFunction instanceof BuiltInTableFunction, functionName + " is not a table-valued function"); + } List argumentList = new ArrayList<>(); argumentList.addAll(List.of(arguments)); if (BITMAP_SCALAR_FUNCTIONS.contains(functionName.toLowerCase(Locale.ROOT))) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java index 31ee33c108..a983a33768 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java @@ -111,6 +111,7 @@ private static ImmutableMap FunctionCatalog.resolve("coalesce", argumentsCount).orElseThrow()) .put("is null", argumentsCount -> FunctionCatalog.resolve("isNull", argumentsCount).orElseThrow()) .put("is not null", argumentsCount -> FunctionCatalog.resolve("notNull", argumentsCount).orElseThrow()) + .put("range", argumentsCount -> FunctionCatalog.resolve("range", argumentsCount).orElseThrow()) .put("__pattern_for_like", argumentsCount -> FunctionCatalog.resolve("patternForLike", argumentsCount).orElseThrow()) .put("__internal_array", argumentsCount -> FunctionCatalog.resolve("array", argumentsCount).orElseThrow()) .build(); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index cfbb11fb9e..a1d3e50dd4 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.ddl.DdlQueryFactory; import com.apple.foundationdb.relational.api.ddl.MetadataOperationsFactory; @@ -44,7 +45,6 @@ import com.apple.foundationdb.relational.recordlayer.query.ProceduralPlan; import com.apple.foundationdb.relational.recordlayer.query.QueryPlan; import com.apple.foundationdb.relational.recordlayer.query.SemanticAnalyzer; -import com.apple.foundationdb.relational.recordlayer.query.StringTrieNode; import com.apple.foundationdb.relational.recordlayer.query.functions.SqlFunctionCatalog; import com.apple.foundationdb.relational.recordlayer.query.functions.SqlFunctionCatalogImpl; import com.apple.foundationdb.relational.util.Assert; @@ -210,12 +210,17 @@ protected String normalizeString(@Nonnull final String value) { @Nonnull public Expression resolveFunction(@Nonnull String functionName, @Nonnull Expression... arguments) { - return getSemanticAnalyzer().resolveFunction(functionName, true, arguments); + return getSemanticAnalyzer().resolveFunction(functionName, true, false, arguments); } @Nonnull public Expression resolveFunction(@Nonnull String functionName, boolean flattenSingleItemRecords, @Nonnull Expression... arguments) { - return getSemanticAnalyzer().resolveFunction(functionName, flattenSingleItemRecords, arguments); + return getSemanticAnalyzer().resolveFunction(functionName, flattenSingleItemRecords, false, arguments); + } + + @Nonnull + public Expression resolveTableValuedFunction(@Nonnull String functionName, @Nonnull Expression... arguments) { + return getSemanticAnalyzer().resolveFunction(functionName, true, true, arguments); } @Override @@ -467,6 +472,16 @@ public LogicalOperator visitNamedQuery(RelationalParser.NamedQueryContext ctx) { return queryVisitor.visitNamedQuery(ctx); } + @Override + public Expression visitTableFunction(@Nonnull final RelationalParser.TableFunctionContext ctx) { + return expressionVisitor.visitTableFunction(ctx); + } + + @Override + public Identifier visitTableFunctionName(final RelationalParser.TableFunctionNameContext ctx) { + return identifierVisitor.visitTableFunctionName(ctx); + } + @Nonnull @Override public Expression visitContinuation(RelationalParser.ContinuationContext ctx) { @@ -557,6 +572,17 @@ public LogicalOperator visitSubqueryTableItem(@Nonnull RelationalParser.Subquery return queryVisitor.visitSubqueryTableItem(ctx); } + @Nonnull + @Override + public LogicalOperator visitInlineTableItem(@Nonnull RelationalParser.InlineTableItemContext ctx) { + return queryVisitor.visitInlineTableItem(ctx); + } + + @Override + public LogicalOperator visitTableValuedFunction(@Nonnull final RelationalParser.TableValuedFunctionContext ctx) { + return queryVisitor.visitTableValuedFunction(ctx); + } + @Nonnull @Override public Set visitIndexHint(@Nonnull RelationalParser.IndexHintContext ctx) { @@ -569,6 +595,12 @@ public Object visitIndexHintType(@Nonnull RelationalParser.IndexHintTypeContext return visitChildren(ctx); } + @Nonnull + @Override + public NonnullPair visitInlineTableDefinition(final RelationalParser.InlineTableDefinitionContext ctx) { + return expressionVisitor.visitInlineTableDefinition(ctx); + } + @Nonnull @Override public Object visitInnerJoin(@Nonnull RelationalParser.InnerJoinContext ctx) { @@ -1064,24 +1096,24 @@ public Object visitLengthTwoOptionalDimension(@Nonnull RelationalParser.LengthTw @Nonnull @Override public List visitUidList(@Nonnull RelationalParser.UidListContext ctx) { - return List.of(); + return identifierVisitor.visitUidList(ctx); } @Nonnull @Override - public NonnullPair visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx) { - return expressionVisitor.visitUidWithNestings(ctx); + public Object visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx) { + return visitChildren(ctx); } @Nonnull @Override - public StringTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx) { + public CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx) { return expressionVisitor.visitUidListWithNestingsInParens(ctx); } @Nonnull @Override - public StringTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx) { + public CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx) { return expressionVisitor.visitUidListWithNestings(ctx); } @@ -1115,6 +1147,12 @@ public Expression visitRecordConstructorForInsert(@Nonnull RelationalParser.Reco return expressionVisitor.visitRecordConstructorForInsert(ctx); } + @Nonnull + @Override + public Expression visitRecordConstructorForInlineTable(final RelationalParser.RecordConstructorForInlineTableContext ctx) { + return expressionVisitor.visitRecordConstructorForInlineTable(ctx); + } + @Nonnull @Override public Expression visitRecordConstructor(@Nonnull RelationalParser.RecordConstructorContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index 4671921f8c..bb67e6f20a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.generated.RelationalParser; @@ -35,7 +36,6 @@ import com.apple.foundationdb.relational.recordlayer.query.OrderByExpression; import com.apple.foundationdb.relational.recordlayer.query.ProceduralPlan; import com.apple.foundationdb.relational.recordlayer.query.QueryPlan; -import com.apple.foundationdb.relational.recordlayer.query.StringTrieNode; import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.RuleNode; @@ -307,6 +307,16 @@ public LogicalOperator visitNamedQuery(@Nonnull RelationalParser.NamedQueryConte return getDelegate().visitNamedQuery(ctx); } + @Override + public Expression visitTableFunction(final RelationalParser.TableFunctionContext ctx) { + return getDelegate().visitTableFunction(ctx); + } + + @Override + public Identifier visitTableFunctionName(final RelationalParser.TableFunctionNameContext ctx) { + return getDelegate().visitTableFunctionName(ctx); + } + @Nonnull @Override public Expression visitContinuation(@Nonnull RelationalParser.ContinuationContext ctx) { @@ -397,6 +407,17 @@ public LogicalOperator visitSubqueryTableItem(@Nonnull RelationalParser.Subquery return getDelegate().visitSubqueryTableItem(ctx); } + @Nonnull + @Override + public LogicalOperator visitInlineTableItem(@Nonnull final RelationalParser.InlineTableItemContext ctx) { + return getDelegate().visitInlineTableItem(ctx); + } + + @Override + public LogicalOperator visitTableValuedFunction(@Nonnull final RelationalParser.TableValuedFunctionContext ctx) { + return getDelegate().visitTableValuedFunction(ctx); + } + @Nonnull @Override public Set visitIndexHint(@Nonnull RelationalParser.IndexHintContext ctx) { @@ -409,6 +430,13 @@ public Object visitIndexHintType(@Nonnull RelationalParser.IndexHintTypeContext return getDelegate().visitIndexHintType(ctx); } + @Nonnull + @Override + public NonnullPair visitInlineTableDefinition(@Nonnull RelationalParser.InlineTableDefinitionContext ctx) { + return getDelegate().visitInlineTableDefinition(ctx); + } + + @Nonnull @Override public Object visitInnerJoin(@Nonnull RelationalParser.InnerJoinContext ctx) { @@ -915,19 +943,19 @@ public List visitUidList(@Nonnull RelationalParser.UidListContext ct @Nonnull @Override - public NonnullPair visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx) { + public Object visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx) { return getDelegate().visitUidWithNestings(ctx); } @Nonnull @Override - public StringTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx) { + public CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx) { return getDelegate().visitUidListWithNestingsInParens(ctx); } @Nonnull @Override - public StringTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx) { + public CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx) { return getDelegate().visitUidListWithNestings(ctx); } @@ -961,6 +989,12 @@ public Expression visitRecordConstructorForInsert(@Nonnull RelationalParser.Reco return getDelegate().visitRecordConstructorForInsert(ctx); } + @Nonnull + @Override + public Expression visitRecordConstructorForInlineTable(@Nonnull RelationalParser.RecordConstructorForInlineTableContext ctx) { + return getDelegate().visitRecordConstructorForInlineTable(ctx); + } + @Nonnull @Override public Expression visitRecordConstructor(@Nonnull RelationalParser.RecordConstructorContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index 42c7011c06..a005a3d87b 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -23,11 +23,13 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.AbstractArrayConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.BooleanValue; import com.apple.foundationdb.record.query.plan.cascades.values.ConditionSelectorValue; import com.apple.foundationdb.record.query.plan.cascades.values.ExistsValue; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; import com.apple.foundationdb.record.query.plan.cascades.values.NullValue; import com.apple.foundationdb.record.query.plan.cascades.values.PickValue; @@ -36,10 +38,14 @@ import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerColumn; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.recordlayer.query.Expression; import com.apple.foundationdb.relational.recordlayer.query.Expressions; +import com.apple.foundationdb.relational.recordlayer.query.Identifier; import com.apple.foundationdb.relational.recordlayer.query.LogicalPlanFragment; import com.apple.foundationdb.relational.recordlayer.query.OrderByExpression; import com.apple.foundationdb.relational.recordlayer.query.ParseHelpers; @@ -57,8 +63,11 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -78,6 +87,14 @@ public static ExpressionVisitor of(@Nonnull BaseVisitor baseVisitor) { return new ExpressionVisitor(baseVisitor); } + @Override + public Expression visitTableFunction(@Nonnull RelationalParser.TableFunctionContext ctx) { + final var functionName = visitTableFunctionName(ctx.tableFunctionName()).toString(); + return ctx.functionArgs() == null + ? getDelegate().resolveTableValuedFunction(functionName) + : getDelegate().resolveTableValuedFunction(functionName, visitFunctionArgs(ctx.functionArgs()).asList().toArray(new Expression[0])); + } + @Nonnull @Override public Expression visitContinuation(@Nonnull RelationalParser.ContinuationContext ctx) { @@ -161,6 +178,38 @@ public OrderByExpression visitOrderByExpression(@Nonnull RelationalParser.OrderB return OrderByExpression.of(expression, descending, nullsLast); } + @Nonnull + @Override + public NonnullPair visitInlineTableDefinition(@Nonnull RelationalParser.InlineTableDefinitionContext ctx) { + final var tableId = visitTableName(ctx.tableName()); + final var columnIdTrie = visitUidListWithNestingsInParens(ctx.uidListWithNestingsInParens()); + int columnCount = Objects.requireNonNull(columnIdTrie.getThis().getChildrenMap()).size(); + final var columnsList = new ArrayList<>(Collections.nCopies(columnCount, (RecordLayerColumn) null)); + for (final var entry : columnIdTrie.getThis().getChildrenMap().entrySet()) { + final var column = toColumn(entry.getKey(), entry.getValue()); + columnsList.set(column.getIndex(), column); + } + final var tableBuilder = RecordLayerTable.newBuilder(false).setName(tableId.getName()); + columnsList.forEach(tableBuilder::addColumn); + return NonnullPair.of(tableId.getName(), columnIdTrie); + } + + private static RecordLayerColumn toColumn(@Nonnull FieldValue.ResolvedAccessor field, @Nonnull CompatibleTypeEvolutionPredicate.FieldAccessTrieNode columnIdTrie) { + final var columnName = field.getName(); + final var builder = RecordLayerColumn.newBuilder().setName(columnName).setIndex(field.getOrdinal()); + if (columnIdTrie.getChildrenMap() == null) { + return builder.setDataType(DataTypeUtils.toRelationalType(field.getType())).build(); + } + int columnCount = columnIdTrie.getChildrenMap().size(); + final var fields = new ArrayList<>(Collections.nCopies(columnCount, (DataType.StructType.Field) null)); + for (final var child : columnIdTrie.getChildrenMap().entrySet()) { + final var column = toColumn(child.getKey(), child.getValue()); + fields.set(column.getIndex(), DataType.StructType.Field.from(column.getName(), column.getDataType(), column.getIndex())); + } + builder.setDataType(DataType.StructType.from(columnName, fields, true)); + return builder.build(); + } + @Nonnull @Override public Expressions visitGroupByClause(@Nonnull RelationalParser.GroupByClauseContext groupByClauseContext) { @@ -602,39 +651,40 @@ public Expression visitNullConstant(@Nonnull RelationalParser.NullConstantContex @Nonnull @Override - public StringTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx) { + public CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestingsInParens(@Nonnull final RelationalParser.UidListWithNestingsInParensContext ctx) { return visitUidListWithNestings(ctx.uidListWithNestings()); } @Nonnull @Override - public StringTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx) { - final var uidMap = - ctx.uidWithNestings() - .stream() - .map(this::visitUidWithNestings) - .collect(ImmutableMap.toImmutableMap(pair -> Assert.notNullUnchecked(pair).getLeft(), - pair -> Assert.notNullUnchecked(pair).getRight(), + public CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestings(@Nonnull final RelationalParser.UidListWithNestingsContext ctx) { + final var uidMap = Streams.mapWithIndex(ctx.uidWithNestings().stream(), + (ctxWithNesting, index) -> { + final var uid = visitUid(ctxWithNesting.uid()); + final var accessor = FieldValue.ResolvedAccessor.of(uid.getName(), (int)index, Type.any()); + if (ctxWithNesting.uidListWithNestingsInParens() == null) { + return NonnullPair.of(accessor, CompatibleTypeEvolutionPredicate.FieldAccessTrieNode.of(Type.any(), null)); + } else { + return NonnullPair.of(accessor, visitUidListWithNestingsInParens(ctxWithNesting.uidListWithNestingsInParens())); + } + }) + .collect(ImmutableMap.toImmutableMap(NonnullPair::getLeft, NonnullPair::getRight, (l, r) -> { - throw Assert.failUnchecked(ErrorCode.AMBIGUOUS_COLUMN, "duplicate column"); + throw Assert.failUnchecked(ErrorCode.AMBIGUOUS_COLUMN, "duplicate column '" + l + "'"); })); - return new StringTrieNode(uidMap); + return CompatibleTypeEvolutionPredicate.FieldAccessTrieNode.of(Type.any(), uidMap); } @Nonnull @Override - public NonnullPair visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx) { - final var uid = visitUid(ctx.uid()); - if (ctx.uidListWithNestingsInParens() == null) { - return NonnullPair.of(uid.getName(), StringTrieNode.leafNode()); - } else { - return NonnullPair.of(uid.getName(), visitUidListWithNestingsInParens(ctx.uidListWithNestingsInParens())); - } + public Expression visitRecordConstructorForInsert(@Nonnull RelationalParser.RecordConstructorForInsertContext ctx) { + final var expressions = parseRecordFieldsUnderReorderings(ctx.expressionWithOptionalName()); + return Expression.ofUnnamed(RecordConstructorValue.ofColumns(expressions.underlyingAsColumns())); } @Nonnull @Override - public Expression visitRecordConstructorForInsert(@Nonnull RelationalParser.RecordConstructorForInsertContext ctx) { + public Expression visitRecordConstructorForInlineTable(@Nonnull RelationalParser.RecordConstructorForInlineTableContext ctx) { final var expressions = parseRecordFieldsUnderReorderings(ctx.expressionWithOptionalName()); return Expression.ofUnnamed(RecordConstructorValue.ofColumns(expressions.underlyingAsColumns())); } @@ -734,7 +784,14 @@ private Expression parseRecordField(@Nonnull ParserRuleContext parserRuleContext if (fieldType == null) { return expression; } - return coerceIfNecessary(expression, fieldType); + final var coercedExpression = coerceIfNecessary(expression, fieldType); + if (expression.getName().isPresent() && targetField.getFieldNameOptional().isPresent()) { + Assert.thatUnchecked(expression.getName().get().equals(Identifier.of(targetField.getFieldNameOptional().get()))); + } + if (expression.getName().isEmpty() && targetField.getFieldNameOptional().isPresent()) { + return coercedExpression.withName(Identifier.of(targetField.getFieldName())); + } + return coercedExpression; } @Nonnull @@ -776,7 +833,7 @@ private Expressions parseRecordFieldsUnderReorderings(@Nonnull final List visitIndexHint(indexHint).stream()).collect(ImmutableSet.toImmutableSet()); return LogicalOperator.generateAccess(tableIdentifier, tableAlias, requestedIndexes, getDelegate().getSemanticAnalyzer(), @@ -325,6 +330,53 @@ public LogicalOperator visitSubqueryTableItem(@Nonnull RelationalParser.Subquery return selectOperator.withName(alias); } + @Nonnull + @Override + public LogicalOperator visitInlineTableItem(@Nonnull RelationalParser.InlineTableItemContext inlineTableItemContext) { + NonnullPair typeMaybe = null; + if (inlineTableItemContext.inlineTableDefinition() != null) { + typeMaybe = visitInlineTableDefinition(inlineTableItemContext.inlineTableDefinition()); + Assert.thatUnchecked(!inlineTableItemContext.recordConstructorForInlineTable().isEmpty()); + Type type = null; + for (final var inlineTableContext : inlineTableItemContext.recordConstructorForInlineTable()) { + final var rowExpression = getDelegate().getPlanGenerationContext().withDisabledLiteralProcessing(() -> visitRecordConstructorForInlineTable(inlineTableContext)); + type = type == null ? rowExpression.getUnderlying().getResultType() + : Type.maximumType(type, rowExpression.getUnderlying().getResultType()); + } + final var actualInlineTableType = type; + final var inlineTypedWithNames = TypeUtils.setFieldNames(actualInlineTableType, typeMaybe.getRight()); + Assert.thatUnchecked(inlineTypedWithNames.isRecord()); + final var stateBuilder = LogicalPlanFragment.State.newBuilder().withTargetType(inlineTypedWithNames); + getDelegate().getCurrentPlanFragment().setState(stateBuilder.build()); + } + final ImmutableList.Builder rowExpressionBuilder = ImmutableList.builder(); + for (final var inlineTableContext : inlineTableItemContext.recordConstructorForInlineTable()) { + final var rowExpression = visitRecordConstructorForInlineTable(inlineTableContext); + + rowExpressionBuilder.add(rowExpression); + } + final var arguments = Expressions.of(rowExpressionBuilder.build()).asList().toArray(new Expression[0]); + final var arrayOfTuples = getDelegate().resolveFunction("__internal_array", false, arguments); + final var explodeExpression = new ExplodeExpression(arrayOfTuples.getUnderlying()); + final var resultingQuantifier = Quantifier.forEach(Reference.of(explodeExpression)); + var output = Expressions.of(LogicalOperator.convertToExpressions(resultingQuantifier)); + return typeMaybe == null + ? LogicalOperator.newUnnamedOperator(output, resultingQuantifier) + : LogicalOperator.newNamedOperator(Identifier.of(typeMaybe.getLeft()), output, resultingQuantifier); + } + + @Override + public LogicalOperator visitTableValuedFunction(@Nonnull RelationalParser.TableValuedFunctionContext tableValuedFunctionContext) { + final var expression = visitTableFunction(tableValuedFunctionContext.tableFunction()); + final var underlyingValue = expression.getUnderlying(); + final var explodeExpression = new TableFunctionExpression(Assert.castUnchecked(underlyingValue, StreamingValue.class)); + final var resultingQuantifier = Quantifier.forEach(Reference.of(explodeExpression)); + final var output = Expressions.of(LogicalOperator.convertToExpressions(resultingQuantifier)); + final var aliasMaybe = Optional.ofNullable(tableValuedFunctionContext.uid() == null ? null : visitUid(tableValuedFunctionContext.uid())); + return aliasMaybe.map(alias -> LogicalOperator.newNamedOperator(alias, output, resultingQuantifier)) + .orElse(LogicalOperator.newUnnamedOperator(output, resultingQuantifier)); + } + @Nonnull @Override public Set visitIndexHint(@Nonnull RelationalParser.IndexHintContext indexHintContext) { @@ -357,7 +409,7 @@ public LogicalOperator visitInsertStatement(@Nonnull RelationalParser.InsertStat } else { final var stateBuilder = LogicalPlanFragment.State.newBuilder().withTargetType(targetType); if (ctx.columns != null) { - stateBuilder.withTargetTypeReorderings(visitUidListWithNestingsInParens(ctx.columns)); + stateBuilder.withTargetTypeReorderings(toString(visitUidListWithNestingsInParens(ctx.columns))); } getDelegate().getCurrentPlanFragment().setState(stateBuilder.build()); insertSource = Assert.castUnchecked(ctx.insertStatementValue().accept(this), LogicalOperator.class); @@ -367,6 +419,15 @@ public LogicalOperator visitInsertStatement(@Nonnull RelationalParser.InsertStat return resultingInsert; } + @Nonnull + private static StringTrieNode toString(@Nonnull CompatibleTypeEvolutionPredicate.FieldAccessTrieNode fieldAccessTrieNode) { + if (fieldAccessTrieNode.getChildrenMap() == null) { + return StringTrieNode.leafNode(); + } + final var map = fieldAccessTrieNode.getChildrenMap().entrySet().stream().collect(ImmutableMap.toImmutableMap(pair -> pair.getKey().getName(), pair -> toString(pair.getValue()))); + return new StringTrieNode(map); + } + @Nonnull @Override public LogicalOperator visitInsertStatementValueSelect(@Nonnull RelationalParser.InsertStatementValueSelectContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java index 55acc9f7c2..16d2eef5b1 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java @@ -20,6 +20,7 @@ package com.apple.foundationdb.relational.recordlayer.query.visitors; +import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.generated.RelationalParser; @@ -34,7 +35,6 @@ import com.apple.foundationdb.relational.recordlayer.query.OrderByExpression; import com.apple.foundationdb.relational.recordlayer.query.ProceduralPlan; import com.apple.foundationdb.relational.recordlayer.query.QueryPlan; -import com.apple.foundationdb.relational.recordlayer.query.StringTrieNode; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -214,6 +214,12 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override LogicalOperator visitNamedQuery(RelationalParser.NamedQueryContext ctx); + @Override + Expression visitTableFunction(@Nonnull RelationalParser.TableFunctionContext ctx); + + @Override + Identifier visitTableFunctionName(RelationalParser.TableFunctionNameContext ctx); + @Nonnull @Override Expression visitContinuation(RelationalParser.ContinuationContext ctx); @@ -274,6 +280,13 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override LogicalOperator visitSubqueryTableItem(@Nonnull RelationalParser.SubqueryTableItemContext ctx); + @Nonnull + @Override + LogicalOperator visitInlineTableItem(@Nonnull RelationalParser.InlineTableItemContext ctx); + + @Override + LogicalOperator visitTableValuedFunction(@Nonnull RelationalParser.TableValuedFunctionContext ctx); + @Nonnull @Override Set visitIndexHint(@Nonnull RelationalParser.IndexHintContext ctx); @@ -282,6 +295,10 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override Object visitIndexHintType(@Nonnull RelationalParser.IndexHintTypeContext ctx); + @Nonnull + @Override + NonnullPair visitInlineTableDefinition(RelationalParser.InlineTableDefinitionContext ctx); + @Nonnull @Override Object visitInnerJoin(@Nonnull RelationalParser.InnerJoinContext ctx); @@ -616,15 +633,15 @@ public interface TypedVisitor extends RelationalParserVisitor { @Nonnull @Override - NonnullPair visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx); + Object visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx); @Nonnull @Override - StringTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx); + CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx); @Nonnull @Override - StringTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx); + CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx); @Nonnull @Override @@ -646,6 +663,9 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override Expression visitRecordConstructorForInsert(@Nonnull RelationalParser.RecordConstructorForInsertContext ctx); + @Override + Expression visitRecordConstructorForInlineTable(RelationalParser.RecordConstructorForInlineTableContext ctx); + @Nonnull @Override Expression visitRecordConstructor(@Nonnull RelationalParser.RecordConstructorContext ctx); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/TypeUtils.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/TypeUtils.java new file mode 100644 index 0000000000..45f297724d --- /dev/null +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/TypeUtils.java @@ -0,0 +1,82 @@ +/* + * TypeUtils.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package com.apple.foundationdb.relational.recordlayer.util; + +import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; +import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.util.Assert; +import com.google.common.collect.ImmutableList; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Optional; + +public final class TypeUtils { + + @Nonnull + public static Type setFieldNames(@Nonnull final Type input, + @Nonnull final CompatibleTypeEvolutionPredicate.FieldAccessTrieNode fieldAccessTrieNode) { + return setFieldNamesInternal(input, fieldAccessTrieNode); + } + + @Nonnull + // PMD incorrectly thinks that comparing array sizes it deemed to be object reference comparison requiring equals() instead. + @SuppressWarnings("PMD.CompareObjectsWithEquals") + private static Type setFieldNamesInternal(@Nonnull final Type input, + @Nonnull final CompatibleTypeEvolutionPredicate.FieldAccessTrieNode trie) { + if (input.isPrimitive()) { + return input; + } + if (trie.getChildrenMap() != null && trie.getChildrenMap().isEmpty()) { + return input; + } + if (input.isArray()) { + final var array = (Type.Array)input; + return array.withElementType(setFieldNamesInternal(Assert.notNullUnchecked(array.getElementType()), trie)); + } + Assert.thatUnchecked(input.isRecord(), ErrorCode.INCOMPATIBLE_TABLE_ALIAS, + () -> "incompatible type found while renaming. Expected " + Type.Record.class.getSimpleName() + + " got " + input.getJavaClass().getSimpleName()); + final var record = (Type.Record)input; + final var recordFields = record.getFields(); + final var newlyNamedFields = ImmutableList.builder(); + final var fieldAliases = new ArrayList<>(trie.getChildrenMap().keySet()); + Assert.thatUnchecked(fieldAliases.size() == recordFields.size(), ErrorCode.INCOMPATIBLE_TABLE_ALIAS, + () -> "number of record fields mismatch"); + fieldAliases.sort(Comparator.comparingInt(FieldValue.ResolvedAccessor::getOrdinal)); + for (int i = 0; i < recordFields.size(); i++) { + final var fieldAlias = fieldAliases.get(i); + final var recordField = recordFields.get(i); + final var fieldTrie = trie.getChildrenMap().get(fieldAlias); + final var renamedFieldType = setFieldNamesInternal(recordField.getFieldType(), fieldTrie); + final var newField = Type.Record.Field.of(renamedFieldType, Optional.ofNullable(fieldAlias.getName()), + Optional.of(recordField.getFieldIndex())); + newlyNamedFields.add(newField); + } + return record.getName() == null ? Type.Record.fromFieldsWithName(record.getName(), record.isNullable(), newlyNamedFields.build()) + : Type.Record.fromFields(record.isNullable(), newlyNamedFields.build()); + } + +} diff --git a/yaml-tests/src/test/java/YamlIntegrationTests.java b/yaml-tests/src/test/java/YamlIntegrationTests.java index 8443138ba4..c2d1e54e78 100644 --- a/yaml-tests/src/test/java/YamlIntegrationTests.java +++ b/yaml-tests/src/test/java/YamlIntegrationTests.java @@ -258,4 +258,9 @@ public void enumTest(YamlTest.Runner runner) throws Exception { public void uuidTest(YamlTest.Runner runner) throws Exception { runner.runYamsql("uuid.yamsql"); } + + @TestTemplate + public void tableFunctionsTest(YamlTest.Runner runner) throws Exception { + runner.runYamsql("table-functions.yamsql"); + } } diff --git a/yaml-tests/src/test/resources/table-functions.metrics.binpb b/yaml-tests/src/test/resources/table-functions.metrics.binpb new file mode 100644 index 0000000000..9396e20760 Binary files /dev/null and b/yaml-tests/src/test/resources/table-functions.metrics.binpb differ diff --git a/yaml-tests/src/test/resources/table-functions.metrics.yaml b/yaml-tests/src/test/resources/table-functions.metrics.yaml new file mode 100644 index 0000000000..89ee335c7f --- /dev/null +++ b/yaml-tests/src/test/resources/table-functions.metrics.yaml @@ -0,0 +1,136 @@ +table-functions: +- query: EXPLAIN select * from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, + 'bar')) as A(B, C, W(X, Y, Z)) + explain: EXPLODE array((@c6 AS B, @c8 AS C, (@c11 AS X, promote(@c13 AS DOUBLE) + AS Y, @c15 AS Z) AS W), (@c20 AS B, @c22 AS C, (@c25 AS X, @c27 AS Y, @c29 + AS Z) AS W)) + task_count: 62 + task_total_time_ms: 2 + transform_count: 23 + transform_time_ms: 0 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 3 + insert_reused_count: 0 +- query: EXPLAIN select B, C, W from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, + (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + explain: EXPLODE array((@c10 AS B, @c12 AS C, (@c15 AS X, promote(@c17 AS DOUBLE) + AS Y, @c19 AS Z) AS W), (@c24 AS B, @c26 AS C, (@c29 AS X, @c31 AS Y, @c33 + AS Z) AS W)) | MAP (_.B AS B, _.C AS C, _.W AS W) + task_count: 68 + task_total_time_ms: 22 + transform_count: 21 + transform_time_ms: 1 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 4 + insert_reused_count: 0 +- query: EXPLAIN select A.B, C, W from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, + (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + explain: EXPLODE array((@c12 AS B, @c14 AS C, (@c17 AS X, promote(@c19 AS DOUBLE) + AS Y, @c21 AS Z) AS W), (@c26 AS B, @c28 AS C, (@c31 AS X, @c33 AS Y, @c35 + AS Z) AS W)) | MAP (_.B AS B, _.C AS C, _.W AS W) + task_count: 68 + task_total_time_ms: 2 + transform_count: 21 + transform_time_ms: 0 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 4 + insert_reused_count: 0 +- query: EXPLAIN select A.B, C as Q, W.X from values (1, 2.0, (3, 4, 'foo')), (10, + 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + explain: EXPLODE array((@c16 AS B, @c18 AS C, (@c21 AS X, promote(@c23 AS DOUBLE) + AS Y, @c25 AS Z) AS W), (@c30 AS B, @c32 AS C, (@c35 AS X, @c37 AS Y, @c39 + AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X) + task_count: 68 + task_total_time_ms: 2 + transform_count: 21 + transform_time_ms: 0 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 4 + insert_reused_count: 0 +- query: EXPLAIN select * from (select A.B, C as Q, W.X from values (1, 2.0, (3, + 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z))) as u + explain: EXPLODE array((@c20 AS B, @c22 AS C, (@c25 AS X, promote(@c27 AS DOUBLE) + AS Y, @c29 AS Z) AS W), (@c34 AS B, @c36 AS C, (@c39 AS X, @c41 AS Y, @c43 + AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X) + task_count: 100 + task_total_time_ms: 4 + transform_count: 33 + transform_time_ms: 1 + transform_yield_count: 4 + insert_time_ms: 0 + insert_new_count: 5 + insert_reused_count: 0 +- query: EXPLAIN select * from (select A.B, C as Q, W.X from values (1, 2.0, (3, + 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z))) as u where + b < 8 + explain: EXPLODE array((@c20 AS B, @c22 AS C, (@c25 AS X, promote(@c27 AS DOUBLE) + AS Y, @c29 AS Z) AS W), (@c34 AS B, @c36 AS C, (@c39 AS X, @c41 AS Y, @c43 + AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X) | FILTER _.B LESS_THAN + @c68 + task_count: 100 + task_total_time_ms: 8 + transform_count: 33 + transform_time_ms: 3 + transform_yield_count: 4 + insert_time_ms: 1 + insert_new_count: 6 + insert_reused_count: 0 +- query: EXPLAIN select * from range(0, 11, 5) + explain: TF range(promote(@c6 AS LONG), promote(@c8 AS LONG), STEP promote(@c10 + AS LONG)) + task_count: 62 + task_total_time_ms: 1 + transform_count: 23 + transform_time_ms: 0 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 3 + insert_reused_count: 0 +- query: EXPLAIN select * from range(6 - 6, 14 + 6 + 1, 20 - 10) + explain: TF range(promote(@c6 - @c6 AS LONG), promote(@c10 + @c6 + @c14 AS LONG), + STEP promote(@c16 - @c18 AS LONG)) + task_count: 62 + task_total_time_ms: 0 + transform_count: 23 + transform_time_ms: 0 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 3 + insert_reused_count: 0 +- query: EXPLAIN select ID as X from range(3) as Y + explain: TF range(0l, promote(@c8 AS LONG), STEP 1l) | MAP (_.ID AS X) + task_count: 68 + task_total_time_ms: 1 + transform_count: 21 + transform_time_ms: 0 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 4 + insert_reused_count: 0 +- query: EXPLAIN select X.ID as A, Y.ID as B from range(3) as X, range(4) as Y + explain: TF range(0l, promote(@c16 AS LONG), STEP 1l) | FLATMAP q0 -> { TF range(0l, + promote(@c23 AS LONG), STEP 1l) AS q1 RETURN (q0.ID AS A, q1.ID AS B) } + task_count: 108 + task_total_time_ms: 25 + transform_count: 33 + transform_time_ms: 5 + transform_yield_count: 6 + insert_time_ms: 1 + insert_new_count: 10 + insert_reused_count: 0 +- query: EXPLAIN select a.id as x, a.col1 as y, b.id as z from t1 as a, range(a.id) + as b + explain: SCAN(<,>) | FLATMAP q0 -> { TF range(0l, q0.ID, STEP 1l) AS q1 RETURN + (q0.ID AS X, q0.COL1 AS Y, q1.ID AS Z) } + task_count: 131 + task_total_time_ms: 55 + transform_count: 49 + transform_time_ms: 46 + transform_yield_count: 9 + insert_time_ms: 0 + insert_new_count: 8 + insert_reused_count: 0 diff --git a/yaml-tests/src/test/resources/table-functions.yamsql b/yaml-tests/src/test/resources/table-functions.yamsql new file mode 100644 index 0000000000..e2d876c973 --- /dev/null +++ b/yaml-tests/src/test/resources/table-functions.yamsql @@ -0,0 +1,131 @@ +# +# table-functions.yamsql +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2021-2025 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +options: + supported_version: !current_version +--- +schema_template: + create table t1(id bigint, col1 string, primary key(id)) +--- +setup: + steps: + - query: INSERT INTO T1 VALUES + (1, 'a'), + (2, 'b'), + (3, 'c') +--- +test_block: + name: table-functions + preset: single_repetition_ordered + tests: + - + - query: select * from values (42) + - result: [{42}] + - + - query: select * from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + - explain: "EXPLODE array((@c6 AS B, @c8 AS C, (@c11 AS X, promote(@c13 AS DOUBLE) AS Y, @c15 AS Z) AS W), (@c20 AS B, @c22 AS C, (@c25 AS X, @c27 AS Y, @c29 AS Z) AS W))" + - result: [{B: 1, C: 2.0, W: {X: 3, Y: 4.0, Z: 'foo'}}, + {B: 10, C: 90.2, W: {X: 5, Y: 6.0, Z: 'bar'}}] + - + - query: select B, C, W from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + - explain: "EXPLODE array((@c10 AS B, @c12 AS C, (@c15 AS X, promote(@c17 AS DOUBLE) AS Y, @c19 AS Z) AS W), (@c24 AS B, @c26 AS C, (@c29 AS X, @c31 AS Y, @c33 AS Z) AS W)) | MAP (_.B AS B, _.C AS C, _.W AS W)" + - result: [{B: 1, C: 2.0, W: {X: 3, Y: 4.0, Z: 'foo'}}, + {B: 10, C: 90.2, W: {X: 5, Y: 6.0, Z: 'bar'}}] + - + - query: select * from values (1, 2.0, [42, 43, 44]), (11, 3.0, [420, 430, 440]) as A(B, C, W) + - result: [{B: 1, C: 2.0, W: [42, 43, 44]}, + {B: 11, C: 3.0, W: [420, 430, 440]}] + - + - query: select * from values (1, 2.0, [('a', 'b', [1, 2, 3])]), (11, 3.0, [('d', 'e', [10, 20, 30])]) as A(B, C, W(X, Y, Z)) + - result: [{B: 1, C: 2.0, W: [{X: 'a', Y: 'b', Z: [1, 2, 3]}]}, + {B: 11, C: 3.0, W: [{X: 'd', Y: 'e', Z: [10, 20, 30]}]}] + - + - query: select A.B, C, W from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + - explain: "EXPLODE array((@c12 AS B, @c14 AS C, (@c17 AS X, promote(@c19 AS DOUBLE) AS Y, @c21 AS Z) AS W), (@c26 AS B, @c28 AS C, (@c31 AS X, @c33 AS Y, @c35 AS Z) AS W)) | MAP (_.B AS B, _.C AS C, _.W AS W)" + - result: [{B: 1, C: 2.0, W: {X: 3, Y: 4.0, Z: 'foo'}}, + {B: 10, C: 90.2, W: {X: 5, Y: 6.0, Z: 'bar'}}] + - + - query: select A.B, C as Q, W.X from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + - explain: "EXPLODE array((@c16 AS B, @c18 AS C, (@c21 AS X, promote(@c23 AS DOUBLE) AS Y, @c25 AS Z) AS W), (@c30 AS B, @c32 AS C, (@c35 AS X, @c37 AS Y, @c39 AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X)" + - result: [{B: 1, Q: 2.0, X: 3}, + {B: 10, Q: 90.2, X: 5}] + - + - query: select * from (select A.B, C as Q, W.X from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z))) as u + - explain: "EXPLODE array((@c20 AS B, @c22 AS C, (@c25 AS X, promote(@c27 AS DOUBLE) AS Y, @c29 AS Z) AS W), (@c34 AS B, @c36 AS C, (@c39 AS X, @c41 AS Y, @c43 AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X)" + - result: [{B: 1, Q: 2.0, X: 3}, + {B: 10, Q: 90.2, X: 5}] + - + - query: select * from (select A.B, C as Q, W.X from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z))) as u where b < 8 + - explain: "EXPLODE array((@c20 AS B, @c22 AS C, (@c25 AS X, promote(@c27 AS DOUBLE) AS Y, @c29 AS Z) AS W), (@c34 AS B, @c36 AS C, (@c39 AS X, @c41 AS Y, @c43 AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X) | FILTER _.B LESS_THAN @c68" + - result: [{B: 1, Q: 2.0, X: 3}] + - + - query: select * from range(1, 4) + - result: [{ID: 1}, {ID: 2}, {ID: 3}] + - + - query: select * from range(-1) + - error: XXXXX + - + - query: select * from range(-1, 4) + - error: XXXXX + - + - query: select * from range(1, 4, -1) + - error: XXXXX + - + - query: select * from range(1, 4, 0) + - error: XXXXX + - + - query: select * from range(0, 12, 5) + - result: [{ID: 0}, {ID: 5}, {ID: 10}] + - + - query: select * from range(0, 11, 5) + - explain: "TF range(promote(@c6 AS LONG), promote(@c8 AS LONG), STEP promote(@c10 AS LONG))" + - result: [{ID: 0}, {ID: 5}, {ID: 10}] + - + - query: select * from range(6 - 6, 14 + 6 + 1, 20 - 10) + - explain: "TF range(promote(@c6 - @c6 AS LONG), promote(@c10 + @c6 + @c14 AS LONG), STEP promote(@c16 - @c18 AS LONG))" + - result: [{ID: 0}, {ID: 10}, {ID: 20}] + - + - query: select ID as X from range(3) as Y + - explain: "TF range(0l, promote(@c8 AS LONG), STEP 1l) | MAP (_.ID AS X)" + - result: [{X: 0}, {X: 1}, {X: 2}] + - + - query: select X.ID as A, Y.ID as B from range(3) as X, range(4) as Y + - explain: "TF range(0l, promote(@c16 AS LONG), STEP 1l) | FLATMAP q0 -> { TF range(0l, promote(@c23 AS LONG), STEP 1l) AS q1 RETURN (q0.ID AS A, q1.ID AS B) }" + - result: [{A: 0, B: 0}, + {A: 0, B: 1}, + {A: 0, B: 2}, + {A: 0, B: 3}, + {A: 1, B: 0}, + {A: 1, B: 1}, + {A: 1, B: 2}, + {A: 1, B: 3}, + {A: 2, B: 0}, + {A: 2, B: 1}, + {A: 2, B: 2}, + {A: 2, B: 3}] + - + - query: select a.id as x, a.col1 as y, b.id as z from t1 as a, range(a.id) as b + - explain: "SCAN(<,>) | FLATMAP q0 -> { TF range(0l, q0.ID, STEP 1l) AS q1 RETURN (q0.ID AS X, q0.COL1 AS Y, q1.ID AS Z) }" + - result: [{X: 1, Y: 'a', Z: 0}, + {X: 2, Y: 'b', Z: 0}, + {X: 2, Y: 'b', Z: 1}, + {X: 3, Y: 'c', Z: 0}, + {X: 3, Y: 'c', Z: 1}, + {X: 3, Y: 'c', Z: 2}] +...