From ebac74a63bbe81ec38b79120178ec2ba20c8c69c Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Fri, 31 Jan 2025 12:04:48 +0100 Subject: [PATCH 01/16] - fix generated files packages. * add post task that copies .token files to fdb-relational-core/src/main/antlr this makes the IDE able to parse the grammar files correctly. --- .gitignore | 3 +++ gradle/antlr.gradle | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/.gitignore b/.gitignore index 38b253831b..9c87e71f27 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,6 @@ fdb-environment.properties # Docker local support run/ + +# Token files required for parsing Antlr files correctly in the IDE +fdb-relational-core/src/main/antlr/*.tokens diff --git a/gradle/antlr.gradle b/gradle/antlr.gradle index 5bddbf54df..ef9e2ca046 100644 --- a/gradle/antlr.gradle +++ b/gradle/antlr.gradle @@ -21,6 +21,7 @@ apply plugin: 'antlr' generateGrammarSource { + outputDirectory = layout.buildDirectory.dir('generated-src/antlr/main/com/apple/foundationdb/relational/generated').get().asFile maxHeapSize = "128m" arguments += ['-package','com.apple.foundationdb.relational.generated', '-listener','-visitor', '-long-messages'] } @@ -29,3 +30,14 @@ sourceSets.configureEach { var generateGrammarSource = tasks.named(getTaskName("generate", "GrammarSource")) java.srcDir(generateGrammarSource.map { files() }) } + +generateGrammarSource.doLast { + final source = layout.buildDirectory.dir('generated-src/antlr/main/com/apple/foundationdb/relational/generated/') + final tokensFile = "*.tokens" + final dest = layout.projectDirectory.dir('src/main/antlr') + copy { + from source + include tokensFile + into dest + } +} \ No newline at end of file From ef0a454b092c597a91a52e9f0848fc936a34b4ab Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Fri, 31 Jan 2025 15:23:28 +0100 Subject: [PATCH 02/16] WIP. --- .../plan/cascades/values/StreamingValue.java | 40 +++++++++++++++++++ .../plan/plans/RecordQueryExplodePlan.java | 5 +++ .../src/main/antlr/RelationalParser.g4 | 13 +++++- 3 files changed, 56 insertions(+), 2 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java new file mode 100644 index 0000000000..63db8b9f5d --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java @@ -0,0 +1,40 @@ +/* + * StreamValue.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.plans.QueryResult; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +public interface StreamingValue extends Value { + + @Nonnull + RecordCursor evalAsStream(@Nullable final FDBRecordStoreBase store, + @Nonnull final EvaluationContext context, + @Nullable final byte[] continuation, + @Nonnull final ExecuteProperties executeProperties); +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java index b4f0f82b99..1d527c89c9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java @@ -33,6 +33,7 @@ import com.apple.foundationdb.record.provider.common.StoreTimer; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.query.plan.AvailableFields; +import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue; import com.apple.foundationdb.record.query.plan.explain.ExplainPlanVisitor; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; @@ -86,6 +87,10 @@ public RecordCursor executePlan(@Nonnull final @Nonnull final EvaluationContext context, @Nullable final byte[] continuation, @Nonnull final ExecuteProperties executeProperties) { + if (collectionValue instanceof StreamingValue) { + final var streamingValue = (StreamingValue)collectionValue; + return streamingValue.evalAsStream(store, context, continuation, executeProperties); + } final var result = collectionValue.eval(store, context); return RecordCursor.fromList(result == null ? ImmutableList.of() : (List)result, continuation) .map(QueryResult::ofComputed); diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index 973bd18f9a..68fbaea189 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -284,6 +284,7 @@ tableSource // done tableSourceItem // done : tableName (AS? alias=uid)? (indexHint (',' indexHint)* )? #atomTableItem // done | query AS? alias=uid #subqueryTableItem // done + | VALUES recordConstructorForInlineTable tableAlias? #inlineTableItem ; indexHint @@ -296,6 +297,10 @@ indexHintType : JOIN | ORDER BY | GROUP BY ; +tableAlias + : AS? tableName '(' uidList ')' + ; + joinPart : (INNER | CROSS)? JOIN tableSourceItem ( @@ -730,11 +735,15 @@ expressionsWithDefaults ; recordConstructorForInsert - : LEFT_ROUND_BRACKET expressionWithOptionalName (',' expressionWithOptionalName)* RIGHT_ROUND_BRACKET + : '(' expressionWithOptionalName (',' expressionWithOptionalName)* ')' + ; + +recordConstructorForInlineTable + : '(' expressionWithOptionalName (',' expressionWithOptionalName)* ')' ; recordConstructor - : ofTypeClause? LEFT_ROUND_BRACKET (uid DOT STAR | STAR | expressionWithName | expressionWithOptionalName (',' expressionWithOptionalName)*) RIGHT_ROUND_BRACKET + : ofTypeClause? '(' (uid DOT STAR | STAR | expressionWithName | expressionWithOptionalName (',' expressionWithOptionalName)*) ')' ; ofTypeClause From c7711221607474b9003838d076996f4298fe3877 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Tue, 4 Feb 2025 10:21:08 +0000 Subject: [PATCH 03/16] WIP. --- .../query/visitors/DelegatingVisitor.java | 16 ++++++++++++++++ .../query/visitors/ExpressionVisitor.java | 7 +++++++ .../recordlayer/query/visitors/TypedVisitor.java | 9 +++++++++ 3 files changed, 32 insertions(+) diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index 4671921f8c..16034986ee 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -397,6 +397,11 @@ public LogicalOperator visitSubqueryTableItem(@Nonnull RelationalParser.Subquery return getDelegate().visitSubqueryTableItem(ctx); } + @Override + public Object visitInlineTableItem(final RelationalParser.InlineTableItemContext ctx) { + return getDelegate().visitInlineTableItem(ctx); + } + @Nonnull @Override public Set visitIndexHint(@Nonnull RelationalParser.IndexHintContext ctx) { @@ -409,6 +414,11 @@ public Object visitIndexHintType(@Nonnull RelationalParser.IndexHintTypeContext return getDelegate().visitIndexHintType(ctx); } + @Override + public Object visitTableAlias(final RelationalParser.TableAliasContext ctx) { + return getDelegate().visitTableAlias(ctx); + } + @Nonnull @Override public Object visitInnerJoin(@Nonnull RelationalParser.InnerJoinContext ctx) { @@ -961,6 +971,12 @@ public Expression visitRecordConstructorForInsert(@Nonnull RelationalParser.Reco return getDelegate().visitRecordConstructorForInsert(ctx); } + @Nonnull + @Override + public Expression visitRecordConstructorForInlineTable(@Nonnull RelationalParser.RecordConstructorForInlineTableContext ctx) { + return getDelegate().visitRecordConstructorForInlineTable(ctx); + } + @Nonnull @Override public Expression visitRecordConstructor(@Nonnull RelationalParser.RecordConstructorContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index 2468faa999..b26cd714c9 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -638,6 +638,13 @@ public Expression visitRecordConstructorForInsert(@Nonnull RelationalParser.Reco return Expression.ofUnnamed(RecordConstructorValue.ofColumns(expressions.underlyingAsColumns())); } + @Nonnull + @Override + public Expression visitRecordConstructorForInlineTable(@Nonnull final RelationalParser.RecordConstructorForInlineTableContext ctx) { + final var expressions = parseRecordFieldsUnderReorderings(ctx.expressionWithOptionalName()); + return Expression.ofUnnamed(RecordConstructorValue.ofColumns(expressions.underlyingAsColumns())); + } + @Nonnull @Override public Expression visitRecordConstructor(@Nonnull RelationalParser.RecordConstructorContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java index 55acc9f7c2..84bd346bec 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java @@ -274,6 +274,9 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override LogicalOperator visitSubqueryTableItem(@Nonnull RelationalParser.SubqueryTableItemContext ctx); + @Override + Object visitInlineTableItem(final RelationalParser.InlineTableItemContext ctx); + @Nonnull @Override Set visitIndexHint(@Nonnull RelationalParser.IndexHintContext ctx); @@ -282,6 +285,9 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override Object visitIndexHintType(@Nonnull RelationalParser.IndexHintTypeContext ctx); + @Override + Object visitTableAlias(RelationalParser.TableAliasContext ctx); + @Nonnull @Override Object visitInnerJoin(@Nonnull RelationalParser.InnerJoinContext ctx); @@ -646,6 +652,9 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override Expression visitRecordConstructorForInsert(@Nonnull RelationalParser.RecordConstructorForInsertContext ctx); + @Override + Expression visitRecordConstructorForInlineTable(RelationalParser.RecordConstructorForInlineTableContext ctx); + @Nonnull @Override Expression visitRecordConstructor(@Nonnull RelationalParser.RecordConstructorContext ctx); From f96c13c82f56498b0b63f4c3e81e1272ddd3057b Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Mon, 17 Feb 2025 17:53:25 +0100 Subject: [PATCH 04/16] wip. --- .../src/main/antlr/RelationalParser.g4 | 10 ++++---- .../query/visitors/BaseVisitor.java | 19 +++++++++++++++ .../query/visitors/DelegatingVisitor.java | 10 +++++--- .../query/visitors/ExpressionVisitor.java | 23 ++++++++++++++++++- .../query/visitors/QueryVisitor.java | 14 +++++++++++ .../query/visitors/TypedVisitor.java | 7 ++++-- 6 files changed, 72 insertions(+), 11 deletions(-) diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index 68fbaea189..1ee0e6c76d 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -282,9 +282,9 @@ tableSource // done ; tableSourceItem // done - : tableName (AS? alias=uid)? (indexHint (',' indexHint)* )? #atomTableItem // done - | query AS? alias=uid #subqueryTableItem // done - | VALUES recordConstructorForInlineTable tableAlias? #inlineTableItem + : tableName (AS? alias=uid)? (indexHint (',' indexHint)* )? #atomTableItem // done + | query AS? alias=uid #subqueryTableItem // done + | VALUES recordConstructorForInlineTable (',' recordConstructorForInlineTable )* inlineTableDefinition? #inlineTableItem ; indexHint @@ -297,7 +297,7 @@ indexHintType : JOIN | ORDER BY | GROUP BY ; -tableAlias +inlineTableDefinition : AS? tableName '(' uidList ')' ; @@ -743,7 +743,7 @@ recordConstructorForInlineTable ; recordConstructor - : ofTypeClause? '(' (uid DOT STAR | STAR | expressionWithName | expressionWithOptionalName (',' expressionWithOptionalName)*) ')' + : ofTypeClause? '(' (uid DOT STAR | STAR | expressionWithName /* this can be removed */ | expressionWithOptionalName (',' expressionWithOptionalName)*) ')' ; ofTypeClause diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index 6b34d02e7a..2b10baa6cd 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -27,6 +27,7 @@ import com.apple.foundationdb.relational.api.ddl.MetadataOperationsFactory; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.metadata.DataType; +import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; @@ -557,6 +558,12 @@ public LogicalOperator visitSubqueryTableItem(@Nonnull RelationalParser.Subquery return queryVisitor.visitSubqueryTableItem(ctx); } + @Nonnull + @Override + public LogicalOperator visitInlineTableItem(@Nonnull RelationalParser.InlineTableItemContext ctx) { + return queryVisitor.visitInlineTableItem(ctx); + } + @Nonnull @Override public Set visitIndexHint(@Nonnull RelationalParser.IndexHintContext ctx) { @@ -569,6 +576,12 @@ public Object visitIndexHintType(@Nonnull RelationalParser.IndexHintTypeContext return visitChildren(ctx); } + @Nonnull + @Override + public Table visitInlineTableDefinition(final RelationalParser.InlineTableDefinitionContext ctx) { + return expressionVisitor.visitInlineTableDefinition(ctx); + } + @Nonnull @Override public Object visitInnerJoin(@Nonnull RelationalParser.InnerJoinContext ctx) { @@ -1115,6 +1128,12 @@ public Expression visitRecordConstructorForInsert(@Nonnull RelationalParser.Reco return expressionVisitor.visitRecordConstructorForInsert(ctx); } + @Nonnull + @Override + public Expression visitRecordConstructorForInlineTable(final RelationalParser.RecordConstructorForInlineTableContext ctx) { + return expressionVisitor.visitRecordConstructorForInlineTable(ctx); + } + @Nonnull @Override public Expression visitRecordConstructor(@Nonnull RelationalParser.RecordConstructorContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index 16034986ee..70c8e4917a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -24,6 +24,7 @@ import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.metadata.DataType; +import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; @@ -397,8 +398,9 @@ public LogicalOperator visitSubqueryTableItem(@Nonnull RelationalParser.Subquery return getDelegate().visitSubqueryTableItem(ctx); } + @Nonnull @Override - public Object visitInlineTableItem(final RelationalParser.InlineTableItemContext ctx) { + public LogicalOperator visitInlineTableItem(final RelationalParser.InlineTableItemContext ctx) { return getDelegate().visitInlineTableItem(ctx); } @@ -414,11 +416,13 @@ public Object visitIndexHintType(@Nonnull RelationalParser.IndexHintTypeContext return getDelegate().visitIndexHintType(ctx); } + @Nonnull @Override - public Object visitTableAlias(final RelationalParser.TableAliasContext ctx) { - return getDelegate().visitTableAlias(ctx); + public Table visitInlineTableDefinition(@Nonnull RelationalParser.InlineTableDefinitionContext ctx) { + return getDelegate().visitInlineTableDefinition(ctx); } + @Nonnull @Override public Object visitInnerJoin(@Nonnull RelationalParser.InnerJoinContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index b26cd714c9..349f3d0b09 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -36,8 +36,12 @@ import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.api.metadata.DataType; +import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerColumn; +import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.recordlayer.query.Expression; import com.apple.foundationdb.relational.recordlayer.query.Expressions; import com.apple.foundationdb.relational.recordlayer.query.LogicalPlanFragment; @@ -160,6 +164,23 @@ public OrderByExpression visitOrderByExpression(@Nonnull RelationalParser.OrderB return OrderByExpression.of(expression, descending, nullsLast); } + @Nonnull + @Override + public Table visitInlineTableDefinition(@Nonnull RelationalParser.InlineTableDefinitionContext ctx) { + final var tableId = visitTableName(ctx.tableName()); + final var columnIds = visitUidList(ctx.uidList()); + final var tableBuilder = RecordLayerTable.newBuilder(false).setName(tableId.getName()); + for (int i = 0; i < columnIds.size(); i++) { + final var column = RecordLayerColumn.newBuilder() + .setName(columnIds.get(i).getName()) + .setIndex(i) + .setDataType(DataType.UnknownType.instance()) + .build(); + tableBuilder.addColumn(column); + } + return tableBuilder.build(); + } + @Nonnull @Override public Expressions visitGroupByClause(@Nonnull RelationalParser.GroupByClauseContext groupByClauseContext) { @@ -640,7 +661,7 @@ public Expression visitRecordConstructorForInsert(@Nonnull RelationalParser.Reco @Nonnull @Override - public Expression visitRecordConstructorForInlineTable(@Nonnull final RelationalParser.RecordConstructorForInlineTableContext ctx) { + public Expression visitRecordConstructorForInlineTable(@Nonnull RelationalParser.RecordConstructorForInlineTableContext ctx) { final var expressions = parseRecordFieldsUnderReorderings(ctx.expressionWithOptionalName()); return Expression.ofUnnamed(RecordConstructorValue.ofColumns(expressions.underlyingAsColumns())); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java index be1726cd1e..03a7082c4e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java @@ -325,6 +325,20 @@ public LogicalOperator visitSubqueryTableItem(@Nonnull RelationalParser.Subquery return selectOperator.withName(alias); } + @Nonnull + @Override + public LogicalOperator visitInlineTableItem(@Nonnull RelationalParser.InlineTableItemContext inlineTableItemContext) { + final ImmutableList.Builder tableRow = ImmutableList.builder(); + for (final var tupleContext : inlineTableItemContext.recordConstructorForInlineTable()) { + tableRow.add(visitRecordConstructorForInlineTable(tupleContext)); + } + final var arguments = Expressions.of(tableRow.build()).asList().toArray(new Expression[0]); + final var arrayOfTuples = getDelegate().resolveFunction("__internal_array", false, arguments); + final var explodeExpression = new ExplodeExpression(arrayOfTuples.getUnderlying()); + final var resultingQuantifier = Quantifier.forEach(Reference.of(explodeExpression)); + return LogicalOperator.newUnnamedOperator(Expressions.ofSingle(arrayOfTuples), resultingQuantifier); + } + @Nonnull @Override public Set visitIndexHint(@Nonnull RelationalParser.IndexHintContext indexHintContext) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java index 84bd346bec..bb7bd04f69 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java @@ -22,6 +22,7 @@ import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.metadata.DataType; +import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.generated.RelationalParserVisitor; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; @@ -274,8 +275,9 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override LogicalOperator visitSubqueryTableItem(@Nonnull RelationalParser.SubqueryTableItemContext ctx); + @Nonnull @Override - Object visitInlineTableItem(final RelationalParser.InlineTableItemContext ctx); + LogicalOperator visitInlineTableItem(final RelationalParser.InlineTableItemContext ctx); @Nonnull @Override @@ -285,8 +287,9 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override Object visitIndexHintType(@Nonnull RelationalParser.IndexHintTypeContext ctx); + @Nonnull @Override - Object visitTableAlias(RelationalParser.TableAliasContext ctx); + Table visitInlineTableDefinition(RelationalParser.InlineTableDefinitionContext ctx); @Nonnull @Override From cdbfad2e5c18a59e9a93a3a2d786d5802c85f1fa Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Mon, 17 Feb 2025 18:04:04 +0100 Subject: [PATCH 05/16] add test. --- .../yamltests/YamlTestExtension.java | 1 + .../src/test/java/YamlIntegrationTests.java | 5 ++ .../src/test/resources/table-functions.yamsql | 49 +++++++++++++++++++ 3 files changed, 55 insertions(+) create mode 100644 yaml-tests/src/test/resources/table-functions.yamsql diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/YamlTestExtension.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/YamlTestExtension.java index a7adfaa10f..5803147e90 100644 --- a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/YamlTestExtension.java +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/YamlTestExtension.java @@ -30,6 +30,7 @@ import com.apple.foundationdb.relational.yamltests.configs.ShowPlanOnDiff; import com.apple.foundationdb.relational.yamltests.configs.YamlTestConfig; import com.apple.foundationdb.relational.yamltests.server.ExternalServer; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/yaml-tests/src/test/java/YamlIntegrationTests.java b/yaml-tests/src/test/java/YamlIntegrationTests.java index 8e8e2d17c1..2649dd20ac 100644 --- a/yaml-tests/src/test/java/YamlIntegrationTests.java +++ b/yaml-tests/src/test/java/YamlIntegrationTests.java @@ -302,4 +302,9 @@ public void recursiveCte(YamlTest.Runner runner) throws Exception { public void enumTest(YamlTest.Runner runner) throws Exception { runner.runYamsql("enum.yamsql"); } + + @TestTemplate + public void tableFunctionsTest(YamlTest.Runner runner) throws Exception { + runner.runYamsql("table-functions.yamsql"); + } } diff --git a/yaml-tests/src/test/resources/table-functions.yamsql b/yaml-tests/src/test/resources/table-functions.yamsql new file mode 100644 index 0000000000..091829aee1 --- /dev/null +++ b/yaml-tests/src/test/resources/table-functions.yamsql @@ -0,0 +1,49 @@ +# +# standard-tests.yamsql +# +# This source file is part of the FoundationDB open source project +# +# Copyright 2021-2024 Apple Inc. and the FoundationDB project authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +schema_template: + create table t1(id bigint, col1 bigint, col2 bigint, primary key(id)) + create index i1 as select col1 from t1 +--- +setup: + steps: + - query: INSERT INTO T1 + VALUES (1, 10, 1), + (2, 10, 2), + (3, 10, 3), + (4, 10, 4), + (5, 10, 5), + (6, 20, 6), + (7, 20, 7), + (8, 20, 8), + (9, 20, 9), + (10, 20, 10), + (11, 20, 11), + (12, 20, 12), + (13, 20, 13) +--- +test_block: + name: table-functions + tests: + - + - query: select * from values (1, 2.0, 'foo'), (10, 90.2, 'bar') + - result: [{1, 2.0, 'foo'}, + {10, 90.2, 'bar'}] +... From 4183fc8c8258e0c17a24e499c2a0e8c099a541de Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Wed, 19 Feb 2025 17:42:49 +0100 Subject: [PATCH 06/16] WIP. --- .../src/main/antlr/RelationalParser.g4 | 2 +- .../recordlayer/query/LogicalOperator.java | 2 +- .../query/visitors/BaseVisitor.java | 10 ++- .../query/visitors/DelegatingVisitor.java | 8 +- .../query/visitors/ExpressionVisitor.java | 73 +++++++++++++------ .../query/visitors/QueryVisitor.java | 34 +++++++-- .../query/visitors/TypedVisitor.java | 9 ++- yaml-tests/src/test/resources/arrays.yamsql | 4 +- .../src/test/resources/table-functions.yamsql | 60 +++++++++------ 9 files changed, 136 insertions(+), 66 deletions(-) diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index 1ee0e6c76d..9b417229c1 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -298,7 +298,7 @@ indexHintType ; inlineTableDefinition - : AS? tableName '(' uidList ')' + : AS? tableName uidListWithNestingsInParens ; joinPart diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java index beb60dcf77..bd170172d6 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java @@ -266,7 +266,7 @@ private static LogicalOperator generateCorrelatedFieldAccess(@Nonnull Expression } @Nonnull - private static Expressions convertToExpressions(@Nonnull Quantifier quantifier) { + public static Expressions convertToExpressions(@Nonnull Quantifier quantifier) { final ImmutableList.Builder attributesBuilder = ImmutableList.builder(); int colCount = 0; final var columns = quantifier.getFlowedColumns(); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index f19ba29a5c..b1d7c419b8 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -22,6 +22,8 @@ import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.ddl.DdlQueryFactory; import com.apple.foundationdb.relational.api.ddl.MetadataOperationsFactory; @@ -1077,24 +1079,24 @@ public Object visitLengthTwoOptionalDimension(@Nonnull RelationalParser.LengthTw @Nonnull @Override public List visitUidList(@Nonnull RelationalParser.UidListContext ctx) { - return List.of(); + return identifierVisitor.visitUidList(ctx); } @Nonnull @Override - public NonnullPair visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx) { + public NonnullPair visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx) { return expressionVisitor.visitUidWithNestings(ctx); } @Nonnull @Override - public StringTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx) { + public CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx) { return expressionVisitor.visitUidListWithNestingsInParens(ctx); } @Nonnull @Override - public StringTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx) { + public CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx) { return expressionVisitor.visitUidListWithNestings(ctx); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index 70c8e4917a..f903f6f3f5 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -22,6 +22,8 @@ import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.api.metadata.Table; @@ -929,19 +931,19 @@ public List visitUidList(@Nonnull RelationalParser.UidListContext ct @Nonnull @Override - public NonnullPair visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx) { + public NonnullPair visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx) { return getDelegate().visitUidWithNestings(ctx); } @Nonnull @Override - public StringTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx) { + public CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx) { return getDelegate().visitUidListWithNestingsInParens(ctx); } @Nonnull @Override - public StringTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx) { + public CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx) { return getDelegate().visitUidListWithNestings(ctx); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index 200bc734ec..a48926e099 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -23,11 +23,13 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.AbstractArrayConstructorValue; import com.apple.foundationdb.record.query.plan.cascades.values.BooleanValue; import com.apple.foundationdb.record.query.plan.cascades.values.ConditionSelectorValue; import com.apple.foundationdb.record.query.plan.cascades.values.ExistsValue; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; import com.apple.foundationdb.record.query.plan.cascades.values.NullValue; import com.apple.foundationdb.record.query.plan.cascades.values.PickValue; @@ -44,6 +46,7 @@ import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.recordlayer.query.Expression; import com.apple.foundationdb.relational.recordlayer.query.Expressions; +import com.apple.foundationdb.relational.recordlayer.query.Identifier; import com.apple.foundationdb.relational.recordlayer.query.LogicalPlanFragment; import com.apple.foundationdb.relational.recordlayer.query.OrderByExpression; import com.apple.foundationdb.relational.recordlayer.query.ParseHelpers; @@ -61,8 +64,11 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Locale; +import java.util.Objects; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -169,19 +175,34 @@ public OrderByExpression visitOrderByExpression(@Nonnull RelationalParser.OrderB @Override public Table visitInlineTableDefinition(@Nonnull RelationalParser.InlineTableDefinitionContext ctx) { final var tableId = visitTableName(ctx.tableName()); - final var columnIds = visitUidList(ctx.uidList()); - final var tableBuilder = RecordLayerTable.newBuilder(false).setName(tableId.getName()); - for (int i = 0; i < columnIds.size(); i++) { - final var column = RecordLayerColumn.newBuilder() - .setName(columnIds.get(i).getName()) - .setIndex(i) - .setDataType(DataType.UnknownType.instance()) - .build(); - tableBuilder.addColumn(column); + final var columnIdTrie = visitUidListWithNestingsInParens(ctx.uidListWithNestingsInParens()); + int columnCount = Objects.requireNonNull(columnIdTrie.getThis().getChildrenMap()).size(); + final var columnsList = new ArrayList<>(Collections.nCopies(columnCount, (RecordLayerColumn) null)); + for (final var entry : columnIdTrie.getThis().getChildrenMap().entrySet()) { + final var column = toColumn(entry.getKey(), entry.getValue()); + columnsList.set(column.getIndex(), column); } + final var tableBuilder = RecordLayerTable.newBuilder(false).setName(tableId.getName()); + columnsList.forEach(tableBuilder::addColumn); return tableBuilder.build(); } + private static RecordLayerColumn toColumn(@Nonnull FieldValue.ResolvedAccessor field, @Nonnull CompatibleTypeEvolutionPredicate.FieldAccessTrieNode columnIdTrie) { + final var columnName = field.getName(); + final var builder = RecordLayerColumn.newBuilder().setName(columnName).setIndex(field.getOrdinal()); + if (columnIdTrie.getChildrenMap() == null) { + return builder.setDataType(DataTypeUtils.toRelationalType(field.getType())).build(); + } + int columnCount = columnIdTrie.getChildrenMap().size(); + final var fields = new ArrayList<>(Collections.nCopies(columnCount, (DataType.StructType.Field) null)); + for (final var child : columnIdTrie.getChildrenMap().entrySet()) { + final var column = toColumn(child.getKey(), child.getValue()); + fields.set(column.getIndex(), DataType.StructType.Field.from(column.getName(), column.getDataType(), column.getIndex())); + } + builder.setDataType(DataType.StructType.from(columnName, fields, true)); + return builder.build(); + } + @Nonnull @Override public Expressions visitGroupByClause(@Nonnull RelationalParser.GroupByClauseContext groupByClauseContext) { @@ -623,36 +644,39 @@ public Expression visitNullConstant(@Nonnull RelationalParser.NullConstantContex @Nonnull @Override - public StringTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx) { + public CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx) { return visitUidListWithNestings(ctx.uidListWithNestings()); } @Nonnull @Override - public StringTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx) { + public CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx) { final var uidMap = - ctx.uidWithNestings() - .stream() - .map(this::visitUidWithNestings) + Streams.mapWithIndex(ctx.uidWithNestings().stream(), (uidWithNesting, index) -> visitUidWithNestingsWithFieldOrdinal(uidWithNesting, (int)index)) .collect(ImmutableMap.toImmutableMap(pair -> Assert.notNullUnchecked(pair).getLeft(), pair -> Assert.notNullUnchecked(pair).getRight(), (l, r) -> { throw Assert.failUnchecked(ErrorCode.AMBIGUOUS_COLUMN, "duplicate column"); })); - return new StringTrieNode(uidMap); + return CompatibleTypeEvolutionPredicate.FieldAccessTrieNode.of(Type.any(), uidMap); } @Nonnull - @Override - public NonnullPair visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx) { + public NonnullPair visitUidWithNestingsWithFieldOrdinal(@Nonnull RelationalParser.UidWithNestingsContext ctx, int fieldOrdinal) { final var uid = visitUid(ctx.uid()); if (ctx.uidListWithNestingsInParens() == null) { - return NonnullPair.of(uid.getName(), StringTrieNode.leafNode()); + return NonnullPair.of(FieldValue.ResolvedAccessor.of(uid.getName(), fieldOrdinal, Type.any()), CompatibleTypeEvolutionPredicate.FieldAccessTrieNode.of(Type.any(), null)); } else { - return NonnullPair.of(uid.getName(), visitUidListWithNestingsInParens(ctx.uidListWithNestingsInParens())); + return NonnullPair.of(FieldValue.ResolvedAccessor.of(uid.getName(), fieldOrdinal, Type.any()), visitUidListWithNestingsInParens(ctx.uidListWithNestingsInParens())); } } + @Nonnull + @Override + public NonnullPair visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx) { + throw new RuntimeException("should never been called"); + } + @Nonnull @Override public Expression visitRecordConstructorForInsert(@Nonnull RelationalParser.RecordConstructorForInsertContext ctx) { @@ -762,7 +786,14 @@ private Expression parseRecordField(@Nonnull ParserRuleContext parserRuleContext if (fieldType == null) { return expression; } - return coerceIfNecessary(expression, fieldType); + final var coercedExpression = coerceIfNecessary(expression, fieldType); + if (expression.getName().isPresent() && targetField.getFieldNameOptional().isPresent()) { + Assert.thatUnchecked(expression.getName().get().equals(Identifier.of(targetField.getFieldNameOptional().get()))); + } + if (expression.getName().isEmpty() && targetField.getFieldNameOptional().isPresent()) { + return coercedExpression.withName(Identifier.of(targetField.getFieldName())); + } + return coercedExpression; } @Nonnull @@ -804,7 +835,7 @@ private Expressions parseRecordFieldsUnderReorderings(@Nonnull final List tableRow = ImmutableList.builder(); - for (final var tupleContext : inlineTableItemContext.recordConstructorForInlineTable()) { - tableRow.add(visitRecordConstructorForInlineTable(tupleContext)); + Table typeMaybe = null; + if (inlineTableItemContext.inlineTableDefinition() != null) { + typeMaybe = visitInlineTableDefinition(inlineTableItemContext.inlineTableDefinition()); + final var stateBuilder = LogicalPlanFragment.State.newBuilder().withTargetType(((RecordLayerTable)typeMaybe).getType()); + getDelegate().getCurrentPlanFragment().setState(stateBuilder.build()); + } + final ImmutableList.Builder rowExpressionBuilder = ImmutableList.builder(); + for (final var inlineTableContext : inlineTableItemContext.recordConstructorForInlineTable()) { + final var rowExpression = visitRecordConstructorForInlineTable(inlineTableContext); + rowExpressionBuilder.add(rowExpression); } - final var arguments = Expressions.of(tableRow.build()).asList().toArray(new Expression[0]); + final var arguments = Expressions.of(rowExpressionBuilder.build()).asList().toArray(new Expression[0]); final var arrayOfTuples = getDelegate().resolveFunction("__internal_array", false, arguments); final var explodeExpression = new ExplodeExpression(arrayOfTuples.getUnderlying()); final var resultingQuantifier = Quantifier.forEach(Reference.of(explodeExpression)); - return LogicalOperator.newUnnamedOperator(Expressions.ofSingle(arrayOfTuples), resultingQuantifier); + final var output = Expressions.of(LogicalOperator.convertToExpressions(resultingQuantifier)); + return typeMaybe == null + ? LogicalOperator.newUnnamedOperator(output, resultingQuantifier) + : LogicalOperator.newNamedOperator(Identifier.of(typeMaybe.getName()), output, resultingQuantifier); } @Nonnull @@ -371,7 +384,7 @@ public LogicalOperator visitInsertStatement(@Nonnull RelationalParser.InsertStat } else { final var stateBuilder = LogicalPlanFragment.State.newBuilder().withTargetType(targetType); if (ctx.columns != null) { - stateBuilder.withTargetTypeReorderings(visitUidListWithNestingsInParens(ctx.columns)); + stateBuilder.withTargetTypeReorderings(toString(visitUidListWithNestingsInParens(ctx.columns))); } getDelegate().getCurrentPlanFragment().setState(stateBuilder.build()); insertSource = Assert.castUnchecked(ctx.insertStatementValue().accept(this), LogicalOperator.class); @@ -381,6 +394,15 @@ public LogicalOperator visitInsertStatement(@Nonnull RelationalParser.InsertStat return resultingInsert; } + @Nonnull + private static StringTrieNode toString(@Nonnull CompatibleTypeEvolutionPredicate.FieldAccessTrieNode fieldAccessTrieNode) { + if (fieldAccessTrieNode.getChildrenMap() == null) { + return StringTrieNode.leafNode(); + } + final var map = fieldAccessTrieNode.getChildrenMap().entrySet().stream().collect(ImmutableMap.toImmutableMap(pair -> pair.getKey().getName(), pair -> toString(pair.getValue()))); + return new StringTrieNode(map); + } + @Nonnull @Override public LogicalOperator visitInsertStatementValueSelect(@Nonnull RelationalParser.InsertStatementValueSelectContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java index bb7bd04f69..71b87b375e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java @@ -20,6 +20,8 @@ package com.apple.foundationdb.relational.recordlayer.query.visitors; +import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.api.metadata.Table; @@ -35,7 +37,6 @@ import com.apple.foundationdb.relational.recordlayer.query.OrderByExpression; import com.apple.foundationdb.relational.recordlayer.query.ProceduralPlan; import com.apple.foundationdb.relational.recordlayer.query.QueryPlan; -import com.apple.foundationdb.relational.recordlayer.query.StringTrieNode; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -625,15 +626,15 @@ public interface TypedVisitor extends RelationalParserVisitor { @Nonnull @Override - NonnullPair visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx); + NonnullPair visitUidWithNestings(@Nonnull RelationalParser.UidWithNestingsContext ctx); @Nonnull @Override - StringTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx); + CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestingsInParens(@Nonnull RelationalParser.UidListWithNestingsInParensContext ctx); @Nonnull @Override - StringTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx); + CompatibleTypeEvolutionPredicate.FieldAccessTrieNode visitUidListWithNestings(@Nonnull RelationalParser.UidListWithNestingsContext ctx); @Nonnull @Override diff --git a/yaml-tests/src/test/resources/arrays.yamsql b/yaml-tests/src/test/resources/arrays.yamsql index 2f0be34df9..adcaf69e39 100644 --- a/yaml-tests/src/test/resources/arrays.yamsql +++ b/yaml-tests/src/test/resources/arrays.yamsql @@ -19,7 +19,7 @@ --- schema_template: - create table A(pk integer, x integer array, primary key(pk)) + create table A(pk integer, x integer array, z string, primary key(pk)) create table B(pk integer, x string array, primary key(pk)) create table C(pk integer, x double array, primary key(pk)) CREATE TYPE AS STRUCT S(f integer) @@ -34,7 +34,7 @@ test_block: preset: single_repetition_ordered tests: - - - query: INSERT INTO A VALUES (1, [1, 2, 3]), (2, [2, 3, 4]), (3, [3, 4, 5]) + - query: INSERT INTO A VALUES (1, [1, 2, 3]), (2, [2, null, 4]), (3, [3, 4, 5]) - count: 3 - - query: SELECT * FROM A diff --git a/yaml-tests/src/test/resources/table-functions.yamsql b/yaml-tests/src/test/resources/table-functions.yamsql index 091829aee1..55c309086d 100644 --- a/yaml-tests/src/test/resources/table-functions.yamsql +++ b/yaml-tests/src/test/resources/table-functions.yamsql @@ -1,9 +1,9 @@ # -# standard-tests.yamsql +# table-functions.yamsql # # This source file is part of the FoundationDB open source project # -# Copyright 2021-2024 Apple Inc. and the FoundationDB project authors +# Copyright 2021-2025 Apple Inc. and the FoundationDB project authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,33 +17,45 @@ # See the License for the specific language governing permissions and # limitations under the License. ---- -schema_template: - create table t1(id bigint, col1 bigint, col2 bigint, primary key(id)) - create index i1 as select col1 from t1 ---- -setup: - steps: - - query: INSERT INTO T1 - VALUES (1, 10, 1), - (2, 10, 2), - (3, 10, 3), - (4, 10, 4), - (5, 10, 5), - (6, 20, 6), - (7, 20, 7), - (8, 20, 8), - (9, 20, 9), - (10, 20, 10), - (11, 20, 11), - (12, 20, 12), - (13, 20, 13) --- test_block: + connect: "jdbc:embed:/__SYS?schema=CATALOG" name: table-functions tests: - - query: select * from values (1, 2.0, 'foo'), (10, 90.2, 'bar') - result: [{1, 2.0, 'foo'}, {10, 90.2, 'bar'}] -... + - + - query: select * from values (1, 2.0, 'foo'), (10, 90.2, 'bar') as A(B, C, D) + - result: [{B: 1, C: 2.0, D: 'foo'}, + {B: 10, C: 90.2, D: 'bar'}] + - + - query: select * from values (1, 2.0, [42, 43, 44]), (11, 3.0, [420, 430, 440]) + - result: [{1, 2.0, [42, 43, 44]}, + {11, 3.0, [420, 430, 440]}] +# - +# - query: select * from values (1, 2.0, [42, 43, 44]), (11, 3.0, [420, 430, 440]) as A(B, C, W) +# - result: [{B: 1, C: 2.0, W: [42, 43, 44]}, +# {B: 11, C: 3.0, W: [420, 430, 440]}] + - + - query: select * from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + - result: [{B: 1, C: 2.0, W: {X: 3, Y: 4.0, Z: 'foo'}}, + {B: 10, C: 90.2, W: {X: 5, Y: 6.0, Z: 'bar'}}] + - + - query: select B, C, W from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + - result: [{B: 1, C: 2.0, W: {X: 3, Y: 4.0, Z: 'foo'}}, + {B: 10, C: 90.2, W: {X: 5, Y: 6.0, Z: 'bar'}}] + - + - query: select A.B, C, W from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + - result: [{B: 1, C: 2.0, W: {X: 3, Y: 4.0, Z: 'foo'}}, + {B: 10, C: 90.2, W: {X: 5, Y: 6.0, Z: 'bar'}}] + - + - query: select A.B, C as Q, W.X from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + - result: [{B: 1, Q: 2.0, X: 3}, + {B: 10, Q: 90.2, X: 5}] + - + - query: select * from (select A.B, C as Q, W.X from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z))) as u + - result: [{B: 1, Q: 2.0, X: 3}, + {B: 10, Q: 90.2, X: 5}] +... \ No newline at end of file From f65dd04bd298ba4b21b282986b756628e8fac0f9 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Tue, 4 Mar 2025 16:07:50 +0000 Subject: [PATCH 07/16] revert WIP changes. --- yaml-tests/src/test/resources/arrays.yamsql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yaml-tests/src/test/resources/arrays.yamsql b/yaml-tests/src/test/resources/arrays.yamsql index adcaf69e39..2f0be34df9 100644 --- a/yaml-tests/src/test/resources/arrays.yamsql +++ b/yaml-tests/src/test/resources/arrays.yamsql @@ -19,7 +19,7 @@ --- schema_template: - create table A(pk integer, x integer array, z string, primary key(pk)) + create table A(pk integer, x integer array, primary key(pk)) create table B(pk integer, x string array, primary key(pk)) create table C(pk integer, x double array, primary key(pk)) CREATE TYPE AS STRUCT S(f integer) @@ -34,7 +34,7 @@ test_block: preset: single_repetition_ordered tests: - - - query: INSERT INTO A VALUES (1, [1, 2, 3]), (2, [2, null, 4]), (3, [3, 4, 5]) + - query: INSERT INTO A VALUES (1, [1, 2, 3]), (2, [2, 3, 4]), (3, [3, 4, 5]) - count: 3 - - query: SELECT * FROM A From d5c08d87440ce3a1ffe6bfd4f70aed7e1f717b39 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Wed, 5 Mar 2025 14:40:22 +0000 Subject: [PATCH 08/16] fixes. --- .../record/query/plan/plans/RecordQueryExplodePlan.java | 3 ++- yaml-tests/src/test/resources/table-functions.yamsql | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java index 1d527c89c9..0c3d921178 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java @@ -93,7 +93,8 @@ public RecordCursor executePlan(@Nonnull final } final var result = collectionValue.eval(store, context); return RecordCursor.fromList(result == null ? ImmutableList.of() : (List)result, continuation) - .map(QueryResult::ofComputed); + .map(QueryResult::ofComputed) + .skipThenLimit(executeProperties.getSkip(), executeProperties.getReturnedRowLimit()); } @Nonnull diff --git a/yaml-tests/src/test/resources/table-functions.yamsql b/yaml-tests/src/test/resources/table-functions.yamsql index 55c309086d..15a49069ef 100644 --- a/yaml-tests/src/test/resources/table-functions.yamsql +++ b/yaml-tests/src/test/resources/table-functions.yamsql @@ -16,7 +16,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +--- +options: + supported_version: !current_version --- test_block: connect: "jdbc:embed:/__SYS?schema=CATALOG" From 6895145e8e971d0d88331d4a1e8e025111efe402 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Thu, 6 Mar 2025 15:50:56 +0000 Subject: [PATCH 09/16] more tests. --- .../resources/table-functions.metrics.binpb | Bin 0 -> 10357 bytes .../resources/table-functions.metrics.yaml | 116 ++++++++++++++++++ .../src/test/resources/table-functions.yamsql | 18 ++- 3 files changed, 133 insertions(+), 1 deletion(-) create mode 100644 yaml-tests/src/test/resources/table-functions.metrics.binpb create mode 100644 yaml-tests/src/test/resources/table-functions.metrics.yaml diff --git a/yaml-tests/src/test/resources/table-functions.metrics.binpb b/yaml-tests/src/test/resources/table-functions.metrics.binpb new file mode 100644 index 0000000000000000000000000000000000000000..eedd67480b37c83189cfe21c0716a75b2dcc7060 GIT binary patch literal 10357 zcmeI2UrZcD9LFVat*lYW8B>BjjH|Y{=6b!`z0xCaCkI?D2?Ywo!!`u>?iSc|***3a z3D!PXUYbgLQmjul#KeewZ48NtCiPFM_+Zq8G)?r$*3`x(HKuAb{^mD#+#a_e34)98 zz-DG|W;QeT{eABH`z_NRu7`Wjl2e+pe`s7!TdJYwyiX_kyF23Tog}AdO4=e%ks;F< zB`?U@xRN8jke`Twpr7m;GK_tKA8tUWAs7(hR!TPa3EoRp+@5=5HDCSwdPVIX^7$`b zUuE#_oF`J*e2+It7w>9I5ZN^4NuRGdT~Fe@WDwS9PDii_4`@z@g4h=61E3c!4hpde zY!sUz3PRmSl^J!|l(Qq`8FGpvWXRAhT^?1UX=B_}6|)iUnX-N|qngo{KHAG=HC0z~ z(X63O(iXiJA}5F@rxY#P)P(0kQihpP%qSn^Nm|jgLN~;^xvZR4_2H$DvvMYb{Rkgx zvP}5P5Lr`)^(e0?Llz%424q!G(@>7{5WsWqaxY+-EFC(uQWh zMpK%cKFO2Zh@4fTd{&cH9eUQJ2ECl2sTrOOshUrRa`A- z^V;&m~05@vJGT8znx^cj)QWI zAYl)AJ^tw7QssK=TALnEgn@@)3I4(Z2ua|TM6n4Uz%?wk zvp^NO_)`QGdD>Z#%W%n93d&?qgr`FLpkODc$aspF6-E361{WrZ2k8{?5MCP!6NS1T zwv&1{=}=vxbh_WX)E9Pfr^=;1vNoyvsLBg|pwtlfKv(r>7SI_8b{Hpp{eJSCpBxti z?*fZ~x2_|VU-|ue8Uts~)ME_HR+I)o7TkzoDNEj=ggVqLgs8k>acD}k2@j$1K%2)= z+ey6Ue}QPgqsd?)f~&Cz0hGgXB2L8R#6wHVi4FGUgt~K8+rWOf@#I|-PQ@RNs~q=o zMR_>s%fbmJfWnf_?)qN{s;{fThPTzs%w4IdeVAPQ^>K))8(&sOs=$U{SvjVd^Tsd; z;SU810~S9ls)}I{DhUG`5@k_^)}2KaI`@23;Y3_c!g$3Xl`vilK`M}!$kBK=@eKxA z=+dyrV4#(@V2!~*lD3jU-Ob(q%h1D}j|G>u4$Z>ep_$*j`_xZ}=BsdStF$0lOlBUiDJsoXr5Te!B@x%!G0POBZ{1f^15M; z${J4{sWD;6szz_=3QuUh&Zi7(ge8gBIBv>;t=VO4jR!V~t`}y1qjdf5>_b4;KmL65 zAL)unWpqU{=IM$^WpqU{A`V3)V*!VbbVZ~xx?;}BmSad|bS+vSsl)>9(ABp?SBHwz z`YKd*y1GZ|J9J$eU2kyQIS0BRDWh!B5=$kPcxB4M(Se7B z!vh!FFVx-IUM?1PPhD(b`|b=-SHBCelh!?Qcjp)BJ|FuR$6atB?KCp2{Nn=OhH|)D)`#vo`?r(2&?dS9i=oUmn#)(2$HRaz(|Yy z4Hs(&;WgF}cKEx)-}StUweiUpwqM)4|Mnj>Ap2o2 w3td>?U8z*Q1(FPD%M{C9#1S_Jg+w69u)X?lL9v`hs$=FO7AyGTrdRO)2GPCmMF0Q* literal 0 HcmV?d00001 diff --git a/yaml-tests/src/test/resources/table-functions.metrics.yaml b/yaml-tests/src/test/resources/table-functions.metrics.yaml new file mode 100644 index 0000000000..5608bc71bf --- /dev/null +++ b/yaml-tests/src/test/resources/table-functions.metrics.yaml @@ -0,0 +1,116 @@ +table-functions: +- query: EXPLAIN select * from values (1, 2.0, 'foo'), (10, 90.2, 'bar') + explain: EXPLODE array((@c6 AS _0, @c8 AS _1, @c10 AS _2), (@c14 AS _0, @c16 AS + _1, @c18 AS _2)) + task_count: 62 + task_total_time_ms: 2 + transform_count: 23 + transform_time_ms: 0 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 3 + insert_reused_count: 0 +- query: EXPLAIN select * from values (1, 2.0, 'foo'), (10, 90.2, 'bar') as A(B, + C, D) + explain: EXPLODE array((@c6 AS B, @c8 AS C, @c10 AS D), (@c14 AS B, @c16 AS C, + @c18 AS D)) + task_count: 62 + task_total_time_ms: 57 + transform_count: 23 + transform_time_ms: 11 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 3 + insert_reused_count: 0 +- query: EXPLAIN select * from values (1, 2.0, [42, 43, 44]), (11, 3.0, [420, 430, + 440]) + explain: EXPLODE array((@c6 AS _0, @c8 AS _1, array(@c11, @c13, @c15) AS _2), + (@c20 AS _0, @c22 AS _1, array(@c25, @c27, @c29) AS _2)) + task_count: 62 + task_total_time_ms: 57 + transform_count: 23 + transform_time_ms: 11 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 3 + insert_reused_count: 0 +- query: EXPLAIN select * from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, + 'bar')) as A(B, C, W(X, Y, Z)) + explain: EXPLODE array(promote((@c6 AS B, @c8 AS C, (@c11 AS X, @c13 AS Y, @c15 + AS Z) AS W) AS INT AS B, ), (@c20 AS B, @c22 AS C, (@c25 AS X, @c27 AS Y, + @c29 AS Z) AS W)) + task_count: 62 + task_total_time_ms: 57 + transform_count: 23 + transform_time_ms: 12 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 3 + insert_reused_count: 0 +- query: EXPLAIN select B, C, W from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, + (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + explain: EXPLODE array(promote((@c10 AS B, @c12 AS C, (@c15 AS X, @c17 AS Y, @c19 + AS Z) AS W) AS INT AS B, ), (@c24 AS B, @c26 AS C, (@c29 AS X, @c31 AS Y, + @c33 AS Z) AS W)) | MAP (_.B AS B, _.C AS C, _.W AS W) + task_count: 68 + task_total_time_ms: 3 + transform_count: 21 + transform_time_ms: 0 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 4 + insert_reused_count: 0 +- query: EXPLAIN select A.B, C, W from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, + (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + explain: EXPLODE array(promote((@c12 AS B, @c14 AS C, (@c17 AS X, @c19 AS Y, @c21 + AS Z) AS W) AS INT AS B, ), (@c26 AS B, @c28 AS C, (@c31 AS X, @c33 AS Y, + @c35 AS Z) AS W)) | MAP (_.B AS B, _.C AS C, _.W AS W) + task_count: 68 + task_total_time_ms: 4 + transform_count: 21 + transform_time_ms: 0 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 4 + insert_reused_count: 0 +- query: EXPLAIN select A.B, C as Q, W.X from values (1, 2.0, (3, 4, 'foo')), (10, + 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + explain: EXPLODE array(promote((@c16 AS B, @c18 AS C, (@c21 AS X, @c23 AS Y, @c25 + AS Z) AS W) AS INT AS B, ), (@c30 AS B, @c32 AS C, (@c35 AS X, @c37 AS Y, + @c39 AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X) + task_count: 68 + task_total_time_ms: 76 + transform_count: 21 + transform_time_ms: 11 + transform_yield_count: 3 + insert_time_ms: 1 + insert_new_count: 4 + insert_reused_count: 0 +- query: EXPLAIN select * from (select A.B, C as Q, W.X from values (1, 2.0, (3, + 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z))) as u + explain: EXPLODE array(promote((@c20 AS B, @c22 AS C, (@c25 AS X, @c27 AS Y, @c29 + AS Z) AS W) AS INT AS B, ), (@c34 AS B, @c36 AS C, (@c39 AS X, @c41 AS Y, + @c43 AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X) + task_count: 100 + task_total_time_ms: 79 + transform_count: 33 + transform_time_ms: 12 + transform_yield_count: 4 + insert_time_ms: 1 + insert_new_count: 5 + insert_reused_count: 0 +- query: EXPLAIN select * from (select A.B, C as Q, W.X from values (1, 2.0, (3, + 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z))) as u where + b < 8 + explain: EXPLODE array(promote((@c20 AS B, @c22 AS C, (@c25 AS X, @c27 AS Y, @c29 + AS Z) AS W) AS INT AS B, ), (@c34 AS B, @c36 AS C, (@c39 AS X, @c41 AS Y, + @c43 AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X) | FILTER _.B LESS_THAN + @c68 + task_count: 100 + task_total_time_ms: 80 + transform_count: 33 + transform_time_ms: 15 + transform_yield_count: 4 + insert_time_ms: 1 + insert_new_count: 6 + insert_reused_count: 0 diff --git a/yaml-tests/src/test/resources/table-functions.yamsql b/yaml-tests/src/test/resources/table-functions.yamsql index 15a49069ef..039c4c8ea5 100644 --- a/yaml-tests/src/test/resources/table-functions.yamsql +++ b/yaml-tests/src/test/resources/table-functions.yamsql @@ -26,38 +26,54 @@ test_block: tests: - - query: select * from values (1, 2.0, 'foo'), (10, 90.2, 'bar') + - explain: "EXPLODE array((@c6 AS _0, @c8 AS _1, @c10 AS _2), (@c14 AS _0, @c16 AS _1, @c18 AS _2))" - result: [{1, 2.0, 'foo'}, {10, 90.2, 'bar'}] - - query: select * from values (1, 2.0, 'foo'), (10, 90.2, 'bar') as A(B, C, D) + - explain: "EXPLODE array((@c6 AS B, @c8 AS C, @c10 AS D), (@c14 AS B, @c16 AS C, @c18 AS D))" - result: [{B: 1, C: 2.0, D: 'foo'}, {B: 10, C: 90.2, D: 'bar'}] - - query: select * from values (1, 2.0, [42, 43, 44]), (11, 3.0, [420, 430, 440]) + - explain: "EXPLODE array((@c6 AS _0, @c8 AS _1, array(@c11, @c13, @c15) AS _2), (@c20 AS _0, @c22 AS _1, array(@c25, @c27, @c29) AS _2))" - result: [{1, 2.0, [42, 43, 44]}, {11, 3.0, [420, 430, 440]}] +# due to: https://github.com/FoundationDB/fdb-record-layer/issues/3231 # - # - query: select * from values (1, 2.0, [42, 43, 44]), (11, 3.0, [420, 430, 440]) as A(B, C, W) # - result: [{B: 1, C: 2.0, W: [42, 43, 44]}, # {B: 11, C: 3.0, W: [420, 430, 440]}] - - query: select * from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + - explain: "EXPLODE array(promote((@c6 AS B, @c8 AS C, (@c11 AS X, @c13 AS Y, @c15 AS Z) AS W) AS INT AS B, ), (@c20 AS B, @c22 AS C, (@c25 AS X, @c27 AS Y, @c29 AS Z) AS W))" - result: [{B: 1, C: 2.0, W: {X: 3, Y: 4.0, Z: 'foo'}}, {B: 10, C: 90.2, W: {X: 5, Y: 6.0, Z: 'bar'}}] - - query: select B, C, W from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + - explain: "EXPLODE array(promote((@c10 AS B, @c12 AS C, (@c15 AS X, @c17 AS Y, @c19 AS Z) AS W) AS INT AS B, ), (@c24 AS B, @c26 AS C, (@c29 AS X, @c31 AS Y, @c33 AS Z) AS W)) | MAP (_.B AS B, _.C AS C, _.W AS W)" - result: [{B: 1, C: 2.0, W: {X: 3, Y: 4.0, Z: 'foo'}}, {B: 10, C: 90.2, W: {X: 5, Y: 6.0, Z: 'bar'}}] - - query: select A.B, C, W from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + - explain: "EXPLODE array(promote((@c12 AS B, @c14 AS C, (@c17 AS X, @c19 AS Y, @c21 AS Z) AS W) AS INT AS B, ), (@c26 AS B, @c28 AS C, (@c31 AS X, @c33 AS Y, @c35 AS Z) AS W)) | MAP (_.B AS B, _.C AS C, _.W AS W)" - result: [{B: 1, C: 2.0, W: {X: 3, Y: 4.0, Z: 'foo'}}, {B: 10, C: 90.2, W: {X: 5, Y: 6.0, Z: 'bar'}}] - - query: select A.B, C as Q, W.X from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) + - explain: "EXPLODE array(promote((@c16 AS B, @c18 AS C, (@c21 AS X, @c23 AS Y, @c25 AS Z) AS W) AS INT AS B, ), (@c30 AS B, @c32 AS C, (@c35 AS X, @c37 AS Y, @c39 AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X)" - result: [{B: 1, Q: 2.0, X: 3}, {B: 10, Q: 90.2, X: 5}] - - query: select * from (select A.B, C as Q, W.X from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z))) as u + - explain: "EXPLODE array(promote((@c20 AS B, @c22 AS C, (@c25 AS X, @c27 AS Y, @c29 AS Z) AS W) AS INT AS B, ), (@c34 AS B, @c36 AS C, (@c39 AS X, @c41 AS Y, @c43 AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X)" - result: [{B: 1, Q: 2.0, X: 3}, {B: 10, Q: 90.2, X: 5}] -... \ No newline at end of file + - + - query: select * from (select A.B, C as Q, W.X from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z))) as u where b < 8 + - explain: "EXPLODE array(promote((@c20 AS B, @c22 AS C, (@c25 AS X, @c27 AS Y, @c29 AS Z) AS W) AS INT AS B, ), (@c34 AS B, @c36 AS C, (@c39 AS X, @c41 AS Y, @c43 AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X) | FILTER _.B LESS_THAN @c68" + - result: [{B: 1, Q: 2.0, X: 3}] + - + - query: select * from values() + - error: "42601" +... From c3c5011a689abec72f19dd91e7f43a44dc756dd0 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Wed, 19 Mar 2025 17:54:01 +0000 Subject: [PATCH 10/16] working version of name aliasing. --- .../plan/cascades/BuiltInTableFunction.java | 42 +++ .../query/plan/cascades/typing/Type.java | 17 + .../plan/cascades/values/RangeValue.java | 333 ++++++++++++++++++ .../plan/cascades/values/StreamingValue.java | 2 +- .../src/main/proto/record_cursor.proto | 6 + .../src/main/proto/record_query_plan.proto | 7 + .../relational/api/exceptions/ErrorCode.java | 3 +- .../src/main/antlr/RelationalParser.g4 | 5 + .../recordlayer/query/LogicalOperator.java | 19 + .../query/visitors/BaseVisitor.java | 14 +- .../query/visitors/DelegatingVisitor.java | 14 +- .../query/visitors/ExpressionVisitor.java | 10 +- .../query/visitors/QueryVisitor.java | 26 +- .../query/visitors/TypedVisitor.java | 9 +- .../recordlayer/util/TypeUtils.java | 59 ++++ .../src/test/java/YamlIntegrationTests.java | 1 + .../resources/table-functions.metrics.binpb | Bin 10357 -> 9650 bytes .../resources/table-functions.metrics.yaml | 85 ++--- .../src/test/resources/table-functions.yamsql | 42 +-- 19 files changed, 603 insertions(+), 91 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInTableFunction.java create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java create mode 100644 fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/TypeUtils.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInTableFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInTableFunction.java new file mode 100644 index 0000000000..4793185329 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInTableFunction.java @@ -0,0 +1,42 @@ +/* + * BuiltInTableFunction.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades; + +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue; +import com.google.common.collect.ImmutableList; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; + +public abstract class BuiltInTableFunction extends BuiltInFunction { + protected BuiltInTableFunction(@Nonnull final String functionName, + @Nonnull final List parameterTypes, + @Nonnull final EncapsulationFunction encapsulationFunction) { + super(functionName, parameterTypes, encapsulationFunction); + } + + protected BuiltInTableFunction(@Nonnull final String functionName, @Nonnull final List parameterTypes, + @Nullable final Type variadicSuffixType, @Nonnull final EncapsulationFunction encapsulationFunction) { + super(functionName, parameterTypes, variadicSuffixType, encapsulationFunction); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java index 2af0b24a4f..d73bad5895 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java @@ -2764,6 +2764,23 @@ public Type getElementType() { return elementType; } + /** + * Return the array with a given element {@link Type} and the same nullability semantics. + * @param elementType The new element type, can be {@code null}. + * @return the array with a given element {@link Type} and the same nullability semantics, if the element type + * matches the current element type, the same instance is returned. + */ + @Nonnull + public Type.Array withElementType(@Nullable final Type elementType) { + if (elementType == null && this.elementType == null) { + return this; + } + if (elementType != null && elementType.equals(this.elementType)) { + return this; + } + return new Array(isNullable(), elementType); + } + /** * Checks whether the array type is erased or not. * diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java new file mode 100644 index 0000000000..27f0fb6083 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java @@ -0,0 +1,333 @@ +/* + * RangeValue.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.RecordCursorContinuation; +import com.apple.foundationdb.record.RecordCursorProto; +import com.apple.foundationdb.record.RecordCursorResult; +import com.apple.foundationdb.record.RecordCursorVisitor; +import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.planprotos.PRangeValue; +import com.apple.foundationdb.record.planprotos.PValue; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; +import com.apple.foundationdb.record.query.plan.cascades.BuiltInTableFunction; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; +import com.apple.foundationdb.record.query.plan.plans.QueryResult; +import com.apple.foundationdb.tuple.ByteArrayUtil2; +import com.google.auto.service.AutoService; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterables; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import java.util.function.Supplier; + +/** + * A Value that is able to return a range between 0 (inclusive) and a given number (inclusive). + */ +@SuppressWarnings("OptionalUsedAsFieldOrParameterType") +public class RangeValue extends AbstractValue implements StreamingValue { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Range-Value"); + + @Nonnull + private final Value endExclusive; + + @Nonnull + private final Optional beginInclusive; + + @Nonnull + private final Optional step; + + public RangeValue(@Nonnull final Value endExclusive, @Nonnull final Optional beginInclusive, + @Nonnull final Optional step) { + this.endExclusive = endExclusive; + this.beginInclusive = beginInclusive; + this.step = step; + } + + @Nonnull + @Override + public RecordCursor evalAsStream(@Nonnull final FDBRecordStoreBase store, + @Nonnull final EvaluationContext context, + @Nullable final byte[] continuation, + @Nonnull final ExecuteProperties executeProperties) { + final long endExclusiveValue = (Long)Verify.verifyNotNull(endExclusive.eval(store, context)); + final var beginInclusiveMaybe = beginInclusive.map(b -> (Long)Verify.verifyNotNull(b.eval(store, context))); + final var stepMaybe = step.map(s -> (Long)Verify.verifyNotNull(s.eval(store, context))); + return new Cursor(store.getExecutor(), endExclusiveValue, beginInclusiveMaybe, stepMaybe, continuation); + } + + @Nonnull + @Override + public ExplainTokensWithPrecedence explain(@Nonnull final Iterable> explainSuppliers) { + return ExplainTokensWithPrecedence.of(new ExplainTokens().addFunctionCall("range")); // todo improve + } + + @Nullable + @Override + public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { + throw new IllegalStateException("unable to eval an aggregation function with eval()"); + } + + @Override + public int hashCodeWithoutChildren() { + return PlanHashable.objectPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH); + } + + @Nonnull + @Override + public PValue toValueProto(@Nonnull final PlanSerializationContext serializationContext) { + return PValue.newBuilder().setRangeValue(toProto(serializationContext)).build(); + } + + @Override + public int planHash(@Nonnull final PlanHashMode hashMode) { + return PlanHashable.objectsPlanHash(hashMode, BASE_HASH, getChildren()); + } + + @Nonnull + @Override + public PRangeValue toProto(@Nonnull final PlanSerializationContext serializationContext) { + final PRangeValue.Builder builder = PRangeValue.newBuilder(); + builder.setEndExclusiveChild(endExclusive.toValueProto(serializationContext)); + beginInclusive.ifPresent(b -> builder.setBeginInclusiveChild(b.toValueProto(serializationContext))); + step.ifPresent(s -> builder.setStepChild(s.toValueProto(serializationContext))); + return builder.build(); + } + + @Nonnull + @Override + protected Iterable computeChildren() { + final ImmutableList.Builder childrenBuilder = ImmutableList.builder(); + childrenBuilder.add(endExclusive); + beginInclusive.ifPresent(childrenBuilder::add); + step.ifPresent(childrenBuilder::add); + return childrenBuilder.build(); + } + + @Nonnull + @Override + public Value withChildren(final Iterable newChildren) { + final var newChildrenSize = Iterables.size(newChildren); + Verify.verify(newChildrenSize <= 3 && newChildrenSize > 0); + final var end = Iterables.get(newChildren, 0); + final Optional begin = newChildrenSize > 1 ? Optional.of(Iterables.get(newChildren, 1)) : Optional.empty(); + final Optional step = newChildrenSize > 2 ? Optional.of(Iterables.get(newChildren, 2)) : Optional.empty(); + if (end != this.endExclusive || begin != this.beginInclusive || step != this.step) { + return new RangeValue(end, begin, step); + } + return this; + } + + @Nonnull + public static RangeValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PRangeValue rangeValueProto) { + final var endExclusive = Value.fromValueProto(serializationContext, rangeValueProto.getEndExclusiveChild()); + final Optional beginInclusive = rangeValueProto.hasBeginInclusiveChild() + ? Optional.of(Value.fromValueProto(serializationContext, rangeValueProto.getBeginInclusiveChild())) + : Optional.empty(); + final Optional step = rangeValueProto.hasStepChild() + ? Optional.of(Value.fromValueProto(serializationContext, rangeValueProto.getStepChild())) + : Optional.empty(); + return new RangeValue(endExclusive, beginInclusive, step); + } + + public static class Cursor implements RecordCursor { + + @Nonnull + private final Executor executor; + + private final long endExclusive; + + @Nonnull + private final Optional step; + + private long nextPosition; // position of the next value to return + + private boolean closed = false; + + public Cursor(@Nonnull final Executor executor, final long endExclusive, @Nonnull final Optional beginInclusive, + @Nonnull final Optional step, @Nullable final byte[] continuation) { + this(executor, endExclusive, step, continuation == null ? beginInclusive.orElse(0L) : Continuation.from(continuation).getNextPosition()); + } + + private Cursor(@Nonnull final Executor executor, final long endExclusive, @Nonnull final Optional step, final long nextPosition) { + this.executor = executor; + this.endExclusive = endExclusive; + this.step = step; + this.nextPosition = nextPosition; + } + + @Nonnull + @Override + public CompletableFuture> onNext() { + return CompletableFuture.completedFuture(getNext()); + } + + @Nonnull + @Override + public RecordCursorResult getNext() { + RecordCursorResult nextResult; + if (nextPosition < endExclusive) { + nextResult = RecordCursorResult.withNextValue(QueryResult.ofComputed(nextPosition), + new Continuation(endExclusive, nextPosition + step.orElse(1L), step)); + nextPosition += step.orElse(1L); + } else { + nextResult = RecordCursorResult.exhausted(); + } + return nextResult; + } + + @Override + public void close() { + closed = true; + } + + @Override + public boolean isClosed() { + return closed; + } + + @Override + public boolean accept(@Nonnull RecordCursorVisitor visitor) { + visitor.visitEnter(this); + return visitor.visitLeave(this); + } + + @Override + @Nonnull + public Executor getExecutor() { + return executor; + } + + private static class Continuation implements RecordCursorContinuation { + private final long endExclusive; + private final long nextPosition; + @Nonnull + private final Optional step; + + public Continuation(long endExclusive, long nextPosition, @Nonnull final Optional step) { + this.nextPosition = nextPosition; + this.endExclusive = endExclusive; + this.step = step; + } + + @Override + public boolean isEnd() { + // If a next value is returned as part of a cursor result, the continuation must not be an end continuation + // (i.e., isEnd() must be false), per the contract of RecordCursorResult. This is the case even if the + // cursor knows for certain that there is no more after that result, as in the ListCursor. + // Concretely, this means that we really need a > here, rather than >=. + return nextPosition > endExclusive; + } + + public long getEndExclusive() { + return endExclusive; + } + + public long getNextPosition() { + return nextPosition; + } + + @Nonnull + public Optional getStep() { + return step; + } + + @Nonnull + @Override + public ByteString toByteString() { + if (isEnd()) { + return ByteString.EMPTY; + } + final var protoBuilder = RecordCursorProto.RangeCursorContinuation.newBuilder() + .setEndExclusive(endExclusive).setNextPosition(nextPosition); + step.ifPresent(protoBuilder::setStep); + return protoBuilder.build().toByteString(); + } + + @Nullable + @Override + public byte[] toBytes() { + return toByteString().toByteArray(); + } + + @Nonnull + public static Continuation from(@Nonnull final RecordCursorProto.RangeCursorContinuation message) { + final var endExclusive = message.getEndExclusive(); + final var nextPosition = message.getNextPosition(); + final Optional step = message.hasStep() ? Optional.of(message.getStep()) : Optional.empty(); + return new Continuation(endExclusive, nextPosition, step); + } + + @Nonnull + public static Continuation from(@Nonnull final byte[] unparsedContinuationBytes) { + try { + final var parsed = RecordCursorProto.RangeCursorContinuation.parseFrom(unparsedContinuationBytes); + return from(parsed); + } catch (InvalidProtocolBufferException ex) { + throw new RecordCoreException("invalid continuation", ex) + .addLogInfo(LogMessageKeys.RAW_BYTES, ByteArrayUtil2.loggable(unparsedContinuationBytes)); + } + } + } + } + + /** + * The {@code range} table function. + */ + @AutoService(BuiltInFunction.class) + public static class RangeFn extends BuiltInTableFunction { + + @Nonnull + private static StreamingValue encapsulateInternal(@Nonnull final BuiltInFunction builtInFunction, + @Nonnull final List arguments) { + Verify.verify(!arguments.isEmpty()); + final var endExclusive = PromoteValue.inject((Value)arguments.get(0), Type.primitiveType(Type.TypeCode.LONG)); + final Optional beginInclusive = arguments.size() > 1 ? Optional.of(PromoteValue.inject((Value)arguments.get(1), Type.primitiveType(Type.TypeCode.LONG))) : Optional.empty(); + final Optional step = arguments.size() > 2 ? Optional.of(PromoteValue.inject((Value)arguments.get(2), Type.primitiveType(Type.TypeCode.LONG))) : Optional.empty(); + return new RangeValue(endExclusive, beginInclusive, step); + } + + public RangeFn() { + super("range", ImmutableList.of(), new Type.Any(), RangeFn::encapsulateInternal); + } + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java index 63db8b9f5d..7fae8e7b06 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java @@ -33,7 +33,7 @@ public interface StreamingValue extends Value { @Nonnull - RecordCursor evalAsStream(@Nullable final FDBRecordStoreBase store, + RecordCursor evalAsStream(@Nonnull final FDBRecordStoreBase store, @Nonnull final EvaluationContext context, @Nullable final byte[] continuation, @Nonnull final ExecuteProperties executeProperties); diff --git a/fdb-record-layer-core/src/main/proto/record_cursor.proto b/fdb-record-layer-core/src/main/proto/record_cursor.proto index c1400c3c8a..8807f3d7d4 100644 --- a/fdb-record-layer-core/src/main/proto/record_cursor.proto +++ b/fdb-record-layer-core/src/main/proto/record_cursor.proto @@ -131,4 +131,10 @@ message RecursiveCursorContinuation { optional bool isInitialState = 1; optional planprotos.PTempTable tempTable = 2; optional bytes activeStateContinuation = 3; +} + +message RangeCursorContinuation { + optional int64 endExclusive = 1; + optional int64 nextPosition = 2; + optional int64 step = 3; } \ No newline at end of file diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 4e93153c64..087f987835 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -244,6 +244,7 @@ message PValue { PCollateValue collate_value = 44; PNumericAggregationValue.PBitmapConstructAgg numeric_aggregation_value_bitmap_construct_agg = 45; PQuantifiedRecordValue quantified_record_value = 46; + PRangeValue range_value = 47; } } @@ -959,6 +960,12 @@ message PCollateValue { optional PValue strength_child = 4; } +message PRangeValue { + optional PValue end_exclusive_child = 1; + optional PValue begin_inclusive_child = 2; + optional PValue step_child = 3; +} + // // Comparisons // diff --git a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/exceptions/ErrorCode.java b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/exceptions/ErrorCode.java index a145bbc2a9..29694190ce 100644 --- a/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/exceptions/ErrorCode.java +++ b/fdb-relational-api/src/main/java/com/apple/foundationdb/relational/api/exceptions/ErrorCode.java @@ -58,7 +58,7 @@ * are in-line with published SQL codes whenever possible). However, often you'll find that the standard does * not have an error code that matches what you're trying to do. In that case, choose a class from * the above table, and then define a unique code. - * Newly introduced error codes follow the pattern "..V..". For example: 08F01 + * Newly introduced error codes follow the pattern "..F..". For example: 08F01 * */ public enum ErrorCode { // Class 00 - Successful Completion @@ -122,6 +122,7 @@ public enum ErrorCode { INVALID_TABLE_DEFINITION("42F16"), UNKNOWN_TYPE("42F18"), INVALID_RECURSION("42F19"), + INCOMPATIBLE_TABLE_ALIAS("42F20"), /** * Indicates that a schema with the given name is already mapped to a schema template. */ diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index 9b417229c1..8b6f298d99 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -226,6 +226,10 @@ namedQuery : name=fullId (columnAliases=fullIdList)? AS? '(' query ')' ; +tableFunction + : functionNameBase '(' functionArgs? ')' inlineTableDefinition? + ; + continuation : WITH CONTINUATION continuationAtom ; @@ -285,6 +289,7 @@ tableSourceItem // done : tableName (AS? alias=uid)? (indexHint (',' indexHint)* )? #atomTableItem // done | query AS? alias=uid #subqueryTableItem // done | VALUES recordConstructorForInlineTable (',' recordConstructorForInlineTable )* inlineTableDefinition? #inlineTableItem + | tableFunction #tableValuedFunction ; indexHint diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java index bd170172d6..340b04a41e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java @@ -56,6 +56,7 @@ import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.util.Assert; +import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; @@ -282,6 +283,24 @@ public static Expressions convertToExpressions(@Nonnull Quantifier quantifier) { return Expressions.of(attributesBuilder.build()); } + @Nonnull + public static Expressions convertToExpressions(@Nonnull final Quantifier quantifier, @Nonnull final Type type) { + Verify.verify(type.isRecord()); + final var recordType = (Type.Record)type; + final ImmutableList.Builder attributesBuilder = ImmutableList.builder(); + int colCount = 0; + final var columns = quantifier.getFlowedColumns(); + for (final var column : columns) { + final var field = recordType.getFields().get(colCount); + final var attributeName = field.getFieldNameOptional().map(Identifier::of); + final var attributeType = DataTypeUtils.toRelationalType(field.getFieldType()); + final var attributeExpression = FieldValue.ofOrdinalNumber(quantifier.getFlowedObjectValue(), colCount); + attributesBuilder.add(new Expression(attributeName, attributeType, attributeExpression)); + colCount++; + } + return Expressions.of(attributesBuilder.build()); + } + @Nonnull public static LogicalOperator generateSelect(@Nonnull Expressions output, @Nonnull LogicalOperators logicalOperators, diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index b1d7c419b8..930240740d 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -29,7 +29,6 @@ import com.apple.foundationdb.relational.api.ddl.MetadataOperationsFactory; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.metadata.DataType; -import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerSchemaTemplate; @@ -47,7 +46,6 @@ import com.apple.foundationdb.relational.recordlayer.query.ProceduralPlan; import com.apple.foundationdb.relational.recordlayer.query.QueryPlan; import com.apple.foundationdb.relational.recordlayer.query.SemanticAnalyzer; -import com.apple.foundationdb.relational.recordlayer.query.StringTrieNode; import com.apple.foundationdb.relational.recordlayer.query.functions.SqlFunctionCatalog; import com.apple.foundationdb.relational.recordlayer.query.functions.SqlFunctionCatalogImpl; import com.apple.foundationdb.relational.util.Assert; @@ -470,6 +468,11 @@ public LogicalOperator visitNamedQuery(RelationalParser.NamedQueryContext ctx) { return queryVisitor.visitNamedQuery(ctx); } + @Override + public Expression visitTableFunction(@Nonnull final RelationalParser.TableFunctionContext ctx) { + return expressionVisitor.visitTableFunction(ctx); + } + @Nonnull @Override public Expression visitContinuation(RelationalParser.ContinuationContext ctx) { @@ -566,6 +569,11 @@ public LogicalOperator visitInlineTableItem(@Nonnull RelationalParser.InlineTabl return queryVisitor.visitInlineTableItem(ctx); } + @Override + public LogicalOperator visitTableValuedFunction(@Nonnull final RelationalParser.TableValuedFunctionContext ctx) { + return queryVisitor.visitTableValuedFunction(ctx); + } + @Nonnull @Override public Set visitIndexHint(@Nonnull RelationalParser.IndexHintContext ctx) { @@ -580,7 +588,7 @@ public Object visitIndexHintType(@Nonnull RelationalParser.IndexHintTypeContext @Nonnull @Override - public Table visitInlineTableDefinition(final RelationalParser.InlineTableDefinitionContext ctx) { + public NonnullPair visitInlineTableDefinition(final RelationalParser.InlineTableDefinitionContext ctx) { return expressionVisitor.visitInlineTableDefinition(ctx); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index f903f6f3f5..dbda2ec38c 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -26,7 +26,6 @@ import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.metadata.DataType; -import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; @@ -38,7 +37,6 @@ import com.apple.foundationdb.relational.recordlayer.query.OrderByExpression; import com.apple.foundationdb.relational.recordlayer.query.ProceduralPlan; import com.apple.foundationdb.relational.recordlayer.query.QueryPlan; -import com.apple.foundationdb.relational.recordlayer.query.StringTrieNode; import org.antlr.v4.runtime.tree.ErrorNode; import org.antlr.v4.runtime.tree.ParseTree; import org.antlr.v4.runtime.tree.RuleNode; @@ -310,6 +308,11 @@ public LogicalOperator visitNamedQuery(@Nonnull RelationalParser.NamedQueryConte return getDelegate().visitNamedQuery(ctx); } + @Override + public Expression visitTableFunction(final RelationalParser.TableFunctionContext ctx) { + return getDelegate().visitTableFunction(ctx); + } + @Nonnull @Override public Expression visitContinuation(@Nonnull RelationalParser.ContinuationContext ctx) { @@ -406,6 +409,11 @@ public LogicalOperator visitInlineTableItem(final RelationalParser.InlineTableIt return getDelegate().visitInlineTableItem(ctx); } + @Override + public LogicalOperator visitTableValuedFunction(final RelationalParser.TableValuedFunctionContext ctx) { + return getDelegate().visitTableValuedFunction(ctx); + } + @Nonnull @Override public Set visitIndexHint(@Nonnull RelationalParser.IndexHintContext ctx) { @@ -420,7 +428,7 @@ public Object visitIndexHintType(@Nonnull RelationalParser.IndexHintTypeContext @Nonnull @Override - public Table visitInlineTableDefinition(@Nonnull RelationalParser.InlineTableDefinitionContext ctx) { + public NonnullPair visitInlineTableDefinition(@Nonnull RelationalParser.InlineTableDefinitionContext ctx) { return getDelegate().visitInlineTableDefinition(ctx); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index a48926e099..cfa64d8d1d 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -39,7 +39,6 @@ import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.metadata.DataType; -import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.metadata.DataTypeUtils; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerColumn; @@ -88,6 +87,11 @@ public static ExpressionVisitor of(@Nonnull BaseVisitor baseVisitor) { return new ExpressionVisitor(baseVisitor); } + @Override + public Expression visitTableFunction(final RelationalParser.TableFunctionContext ctx) { + return super.visitTableFunction(ctx); + } + @Nonnull @Override public Expression visitContinuation(@Nonnull RelationalParser.ContinuationContext ctx) { @@ -173,7 +177,7 @@ public OrderByExpression visitOrderByExpression(@Nonnull RelationalParser.OrderB @Nonnull @Override - public Table visitInlineTableDefinition(@Nonnull RelationalParser.InlineTableDefinitionContext ctx) { + public NonnullPair visitInlineTableDefinition(@Nonnull RelationalParser.InlineTableDefinitionContext ctx) { final var tableId = visitTableName(ctx.tableName()); final var columnIdTrie = visitUidListWithNestingsInParens(ctx.uidListWithNestingsInParens()); int columnCount = Objects.requireNonNull(columnIdTrie.getThis().getChildrenMap()).size(); @@ -184,7 +188,7 @@ public Table visitInlineTableDefinition(@Nonnull RelationalParser.InlineTableDef } final var tableBuilder = RecordLayerTable.newBuilder(false).setName(tableId.getName()); columnsList.forEach(tableBuilder::addColumn); - return tableBuilder.build(); + return NonnullPair.of(tableId.getName(), columnIdTrie); } private static RecordLayerColumn toColumn(@Nonnull FieldValue.ResolvedAccessor field, @Nonnull CompatibleTypeEvolutionPredicate.FieldAccessTrieNode columnIdTrie) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java index ffc96f49b1..ad8cfd165e 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java @@ -34,6 +34,7 @@ import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.generated.RelationalLexer; @@ -52,6 +53,7 @@ import com.apple.foundationdb.relational.recordlayer.query.SemanticAnalyzer; import com.apple.foundationdb.relational.recordlayer.query.StringTrieNode; import com.apple.foundationdb.relational.recordlayer.util.MemoizedFunction; +import com.apple.foundationdb.relational.recordlayer.util.TypeUtils; import com.apple.foundationdb.relational.util.Assert; import com.google.common.collect.ImmutableList; @@ -331,25 +333,41 @@ public LogicalOperator visitSubqueryTableItem(@Nonnull RelationalParser.Subquery @Nonnull @Override public LogicalOperator visitInlineTableItem(@Nonnull RelationalParser.InlineTableItemContext inlineTableItemContext) { - Table typeMaybe = null; + NonnullPair typeMaybe = null; if (inlineTableItemContext.inlineTableDefinition() != null) { typeMaybe = visitInlineTableDefinition(inlineTableItemContext.inlineTableDefinition()); - final var stateBuilder = LogicalPlanFragment.State.newBuilder().withTargetType(((RecordLayerTable)typeMaybe).getType()); + Assert.thatUnchecked(!inlineTableItemContext.recordConstructorForInlineTable().isEmpty()); + Type type = null; + for (final var inlineTableContext : inlineTableItemContext.recordConstructorForInlineTable()) { + final var rowExpression = getDelegate().getPlanGenerationContext().withDisabledLiteralProcessing(() -> visitRecordConstructorForInlineTable(inlineTableContext)); + type = type == null ? rowExpression.getUnderlying().getResultType() + : Type.maximumType(type, rowExpression.getUnderlying().getResultType()); + } + final var actualInlineTableType = type; + final var inlineTypedWithNames = TypeUtils.setFieldNames(actualInlineTableType, typeMaybe.getRight()); + Assert.thatUnchecked(inlineTypedWithNames.isRecord()); + final var stateBuilder = LogicalPlanFragment.State.newBuilder().withTargetType(inlineTypedWithNames); getDelegate().getCurrentPlanFragment().setState(stateBuilder.build()); } final ImmutableList.Builder rowExpressionBuilder = ImmutableList.builder(); for (final var inlineTableContext : inlineTableItemContext.recordConstructorForInlineTable()) { final var rowExpression = visitRecordConstructorForInlineTable(inlineTableContext); + rowExpressionBuilder.add(rowExpression); } final var arguments = Expressions.of(rowExpressionBuilder.build()).asList().toArray(new Expression[0]); final var arrayOfTuples = getDelegate().resolveFunction("__internal_array", false, arguments); final var explodeExpression = new ExplodeExpression(arrayOfTuples.getUnderlying()); final var resultingQuantifier = Quantifier.forEach(Reference.of(explodeExpression)); - final var output = Expressions.of(LogicalOperator.convertToExpressions(resultingQuantifier)); + var output = Expressions.of(LogicalOperator.convertToExpressions(resultingQuantifier)); return typeMaybe == null ? LogicalOperator.newUnnamedOperator(output, resultingQuantifier) - : LogicalOperator.newNamedOperator(Identifier.of(typeMaybe.getName()), output, resultingQuantifier); + : LogicalOperator.newNamedOperator(Identifier.of(typeMaybe.getLeft()), output, resultingQuantifier); + } + + @Override + public LogicalOperator visitTableValuedFunction(@Nonnull RelationalParser.TableValuedFunctionContext tableValuedFunctionContext) { + return null; } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java index 71b87b375e..3da5ceb23f 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java @@ -24,7 +24,6 @@ import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.metadata.DataType; -import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.generated.RelationalParserVisitor; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerIndex; @@ -216,6 +215,9 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override LogicalOperator visitNamedQuery(RelationalParser.NamedQueryContext ctx); + @Override + Expression visitTableFunction(@Nonnull RelationalParser.TableFunctionContext ctx); + @Nonnull @Override Expression visitContinuation(RelationalParser.ContinuationContext ctx); @@ -280,6 +282,9 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override LogicalOperator visitInlineTableItem(final RelationalParser.InlineTableItemContext ctx); + @Override + LogicalOperator visitTableValuedFunction(@Nonnull RelationalParser.TableValuedFunctionContext ctx); + @Nonnull @Override Set visitIndexHint(@Nonnull RelationalParser.IndexHintContext ctx); @@ -290,7 +295,7 @@ public interface TypedVisitor extends RelationalParserVisitor { @Nonnull @Override - Table visitInlineTableDefinition(RelationalParser.InlineTableDefinitionContext ctx); + NonnullPair visitInlineTableDefinition(RelationalParser.InlineTableDefinitionContext ctx); @Nonnull @Override diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/TypeUtils.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/TypeUtils.java new file mode 100644 index 0000000000..edf78ae7da --- /dev/null +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/TypeUtils.java @@ -0,0 +1,59 @@ +package com.apple.foundationdb.relational.recordlayer.util; + +import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; +import com.apple.foundationdb.relational.api.exceptions.ErrorCode; +import com.apple.foundationdb.relational.util.Assert; +import com.google.common.collect.ImmutableList; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Optional; + +public final class TypeUtils { + + @Nonnull + public static Type setFieldNames(@Nonnull final Type input, + @Nonnull final CompatibleTypeEvolutionPredicate.FieldAccessTrieNode fieldAccessTrieNode) { + return setFieldNamesInternal(input, fieldAccessTrieNode); + } + + @Nonnull + private static Type setFieldNamesInternal(@Nonnull final Type input, + @Nonnull final CompatibleTypeEvolutionPredicate.FieldAccessTrieNode trie) { + if (input.isPrimitive()) { + return input; + } + if (trie.getChildrenMap() != null && trie.getChildrenMap().isEmpty()) { + return input; + } + if (input.isArray()) { + final var array = (Type.Array)input; + return array.withElementType(setFieldNamesInternal(Assert.notNullUnchecked(array.getElementType()), trie)); + } + Assert.thatUnchecked(input.isRecord(), ErrorCode.INCOMPATIBLE_TABLE_ALIAS, + () -> "incompatible type found while renaming. Expected " + Type.Record.class.getSimpleName() + + " got " + input.getJavaClass().getSimpleName()); + final var record = (Type.Record)input; + final var recordFields = record.getFields(); + final var newlyNamedFields = ImmutableList.builder(); + final var fieldAliases = new ArrayList<>(trie.getChildrenMap().keySet()); + Assert.thatUnchecked(fieldAliases.size() == recordFields.size(), ErrorCode.INCOMPATIBLE_TABLE_ALIAS, + () -> "number of record fields mismatch"); + fieldAliases.sort(Comparator.comparingInt(FieldValue.ResolvedAccessor::getOrdinal)); + for (int i = 0; i < recordFields.size(); i++) { + final var fieldAlias = fieldAliases.get(i); + final var recordField = recordFields.get(i); + final var fieldTrie = trie.getChildrenMap().get(fieldAlias); + final var renamedFieldType = setFieldNamesInternal(recordField.getFieldType(), fieldTrie); + final var newField = Type.Record.Field.of(renamedFieldType, Optional.ofNullable(fieldAlias.getName()), + Optional.of(recordField.getFieldIndex())); + newlyNamedFields.add(newField); + } + return record.getName() == null ? Type.Record.fromFieldsWithName(record.getName(), record.isNullable(), newlyNamedFields.build()) + : Type.Record.fromFields(record.isNullable(), newlyNamedFields.build()); + } + +} diff --git a/yaml-tests/src/test/java/YamlIntegrationTests.java b/yaml-tests/src/test/java/YamlIntegrationTests.java index 1d2fabc9f2..32ee6efc9f 100644 --- a/yaml-tests/src/test/java/YamlIntegrationTests.java +++ b/yaml-tests/src/test/java/YamlIntegrationTests.java @@ -288,6 +288,7 @@ public void enumTest(YamlTest.Runner runner) throws Exception { } @TestTemplate + @MaintainYamlTestConfig(YamlTestConfigFilters.CORRECT_EXPLAIN_AND_METRICS) public void tableFunctionsTest(YamlTest.Runner runner) throws Exception { runner.runYamsql("table-functions.yamsql"); } diff --git a/yaml-tests/src/test/resources/table-functions.metrics.binpb b/yaml-tests/src/test/resources/table-functions.metrics.binpb index eedd67480b37c83189cfe21c0716a75b2dcc7060..2ada5ca667cf51506bf8f416f9c5c19e4834c1f6 100644 GIT binary patch delta 957 zcmewwu*rLJA5)ItjjPi{B*YbVd^*RX!ECT}ozZ4)W??4A7n9{#)5IHDxx_i_1e#_n zW(2DFy-53UD=HjTAJ;cf-22|3sCIhJA;CVryimyl5ESMb_gyu0$ z7GzeQ?90U?UQm>un_rTu;gD=-tl$`|;Nl|0Jf`OOqqG+q6$}0$u1Lu=CIrP6<(ki$4m1_OE&BV!;Bm|0tttM1YUKi>pFn zQBh*$WI-MkT_lH?BRK@bjMPy8I?ocsjM4;=;gjQdG8msv?iGKxc`L62<76QoWs(hW z+T6{r!N_=J@)m&(xNGO1V+6W(NvjypwJR3!V0CT2Ac||j?t%sz+_grA$gbTZh~nA; zK@`^#WdPK*_a_U9^=qGpgzbxW(?PEN@mCn=+GFcE!LB`y7PeQ#RFERs2qlvLiiuAa zl5(8Kcul7c$pks^vS(A96ABoB1Phsl2w zHtOu-<&xk?5x8`%g&F9KsmuEqf$lhbLCS&+=#Ix|AwrxxT$pxqPdvADbF}htrpW}H N0(AP-%|7ayi~vf{Da8N) delta 1163 zcmdnw{WUDJNAotu!yWBr`v+Sjc|zL{_zlwvuu}TUofoIqU>3zMsk{ zA+B)hi;xDh!H-p}7R(L|LgAA$SyW9Ok_`G=Mx4Fwa0o0mwE3vkk#) z3#ckhsRPWD8~LwJULYtqxr#+Zi+B?z-(#6GAxH2|&rWHeYi>>7!wPiH@uQX-YlWE@ zUrwILlqS){$|V7G@R3#9K}r@)I|EX3_BS3(&E~lVjP$1wp>?^a}w| zPC5!ezfYEzSDwtzA;T=y#RByoA3LeK*E8Olyq}}Z_%bIj++75wtvbmlA*!(En=;Ua zFV1pWumHpTBz6~0F5pzr1UkAypWw?0Jor*`>=6`}&jEpZPXA1Y5+~5SJ1{Z-3uUD%8UG#bD6;MonXpRLt z=`@aTAHu(qMc$xq<)P=Ch(sjPTIIqiu7%IM7M|CZCd63UN}3 zz>-xrR3#J@J~smsAB(}1!$B6Tz^KLLBsqB%VA7IC51@ScOeQHTPQj@h5o13lJIZY| zImQbNpA> Date: Fri, 28 Mar 2025 14:56:12 +0100 Subject: [PATCH 11/16] support table valued functions. --- .../plan/cascades/BuiltInTableFunction.java | 1 - .../query/plan/cascades/PlannerRuleSet.java | 4 +- .../expressions/TableFunctionExpression.java | 168 +++++++++++ .../RelationalExpressionMatchers.java | 6 + .../properties/CardinalitiesProperty.java | 14 + .../properties/DerivationsProperty.java | 10 + .../properties/DistinctRecordsProperty.java | 7 + .../cascades/properties/OrderingProperty.java | 7 + .../properties/PrimaryKeyProperty.java | 7 + .../properties/StoredRecordProperty.java | 7 + .../rules/ImplementTableFunctionRule.java | 52 ++++ .../query/plan/cascades/typing/Type.java | 1 + .../plan/cascades/values/RangeValue.java | 130 ++++++-- .../plan/cascades/values/StreamingValue.java | 11 +- .../plan/explain/ExplainPlanVisitor.java | 8 + .../plans/RecordQueryTableFunctionPlan.java | 280 ++++++++++++++++++ .../src/main/proto/record_query_plan.proto | 8 + .../src/main/antlr/RelationalLexer.g4 | 2 +- .../src/main/antlr/RelationalParser.g4 | 12 +- .../recordlayer/query/LogicalOperator.java | 19 -- .../recordlayer/query/SemanticAnalyzer.java | 9 +- .../functions/SqlFunctionCatalogImpl.java | 1 + .../query/visitors/BaseVisitor.java | 15 +- .../query/visitors/DelegatingVisitor.java | 9 +- .../query/visitors/ExpressionVisitor.java | 7 +- .../query/visitors/IdentifierVisitor.java | 5 + .../query/visitors/QueryVisitor.java | 17 +- .../query/visitors/TypedVisitor.java | 6 +- .../recordlayer/util/TypeUtils.java | 23 ++ .../yamltests/YamlTestExtension.java | 1 - .../src/test/resources/table-functions.yamsql | 64 +++- 31 files changed, 838 insertions(+), 73 deletions(-) create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TableFunctionExpression.java create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTableFunctionRule.java create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInTableFunction.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInTableFunction.java index 4793185329..9b3aad3c09 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInTableFunction.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/BuiltInTableFunction.java @@ -22,7 +22,6 @@ import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue; -import com.google.common.collect.ImmutableList; import javax.annotation.Nonnull; import javax.annotation.Nullable; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java index 90f7be1151..ab5a5ee58a 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/PlannerRuleSet.java @@ -35,6 +35,7 @@ import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementInUnionRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementInsertRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementRecursiveUnionRule; +import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementTableFunctionRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementTempTableInsertRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementIntersectionRule; import com.apple.foundationdb.record.query.plan.cascades.rules.ImplementNestedLoopJoinRule; @@ -179,7 +180,8 @@ public class PlannerRuleSet { new ImplementInsertRule(), new ImplementTempTableInsertRule(), new ImplementUpdateRule(), - new ImplementRecursiveUnionRule() + new ImplementRecursiveUnionRule(), + new ImplementTableFunctionRule() ); private static final List> EXPLORATION_RULES = diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TableFunctionExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TableFunctionExpression.java new file mode 100644 index 0000000000..cee2333511 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TableFunctionExpression.java @@ -0,0 +1,168 @@ +/* + * TableFunctionExpression.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2020 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.expressions; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.ComparisonRange; +import com.apple.foundationdb.record.query.plan.cascades.Compensation; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.IdentityBiMap; +import com.apple.foundationdb.record.query.plan.cascades.MatchInfo; +import com.apple.foundationdb.record.query.plan.cascades.PartialMatch; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.explain.InternalPlannerGraphRewritable; +import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue; +import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.PullUp; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * A table function expression that delegates the actual execution to an underlying {@link StreamingValue} which + * effectively returns a stream of results. + */ +@API(API.Status.EXPERIMENTAL) +public class TableFunctionExpression implements RelationalExpression, InternalPlannerGraphRewritable { + @Nonnull + private final StreamingValue value; + + public TableFunctionExpression(@Nonnull final StreamingValue value) { + this.value = value; + } + + @Nonnull + @Override + public Value getResultValue() { + return new QueriedValue(value.getResultType()); + } + + @Nonnull + @Override + public Set getDynamicTypes() { + return value.getDynamicTypes(); + } + + @Nonnull + public StreamingValue getValue() { + return value; + } + + @Nonnull + @Override + public List getQuantifiers() { + return Collections.emptyList(); + } + + @Nonnull + @Override + public Set getCorrelatedTo() { + return value.getCorrelatedTo(); + } + + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public boolean equalsWithoutChildren(@Nonnull RelationalExpression otherExpression, + @Nonnull final AliasMap equivalencesMap) { + if (this == otherExpression) { + return true; + } + if (!(otherExpression instanceof TableFunctionExpression)) { + return false; + } + + final var otherTableFunctionExpression = (TableFunctionExpression)otherExpression; + + return value.semanticEquals(otherTableFunctionExpression.getValue(), equivalencesMap) && + semanticEqualsForResults(otherExpression, equivalencesMap); + } + + @Override + public int hashCodeWithoutChildren() { + return Objects.hash(value); + } + + @Nonnull + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public TableFunctionExpression translateCorrelations(@Nonnull final TranslationMap translationMap, + final boolean shouldSimplifyValues, + @Nonnull final List translatedQuantifiers) { + final Value translatedCollectionValue = value.translateCorrelations(translationMap, shouldSimplifyValues); + if (translatedCollectionValue != value) { + return new TableFunctionExpression((StreamingValue)translatedCollectionValue); + } + return this; + } + + @Nonnull + @Override + public Iterable subsumedBy(@Nonnull final RelationalExpression candidateExpression, + @Nonnull final AliasMap bindingAliasMap, + @Nonnull final IdentityBiMap partialMatchMap, + @Nonnull final EvaluationContext evaluationContext) { + if (!isCompatiblyAndCompletelyBound(bindingAliasMap, candidateExpression.getQuantifiers())) { + return ImmutableList.of(); + } + + return exactlySubsumedBy(candidateExpression, bindingAliasMap, partialMatchMap, TranslationMap.empty()); + } + + @Nonnull + @Override + public Compensation compensate(@Nonnull final PartialMatch partialMatch, + @Nonnull final Map boundParameterPrefixMap, + @Nullable final PullUp pullUp, + @Nonnull final CorrelationIdentifier nestingAlias) { + // subsumedBy() is based on equality and this expression is always a leaf, thus we return empty here as + // if there is a match, it's exact + return Compensation.noCompensation(); + } + + @Nonnull + @Override + public PlannerGraph rewriteInternalPlannerGraph(@Nonnull final List childGraphs) { + return PlannerGraph.fromNodeAndChildGraphs( + new PlannerGraph.LogicalOperatorNode(this, + "TFunc", + ImmutableList.of(toString()), + ImmutableMap.of()), + childGraphs); + } + + @Override + public String toString() { + return value.toString(); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java index 0fe1401766..3c7b702c48 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/matching/structure/RelationalExpressionMatchers.java @@ -28,6 +28,7 @@ import com.apple.foundationdb.record.query.plan.cascades.expressions.GroupByExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.InsertExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.RecursiveUnionExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TableFunctionExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableInsertExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalDistinctExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalFilterExpression; @@ -246,6 +247,11 @@ public static BindingMatcher explodeExpression() { return ofTypeOwning(ExplodeExpression.class, CollectionMatcher.empty()); } + @Nonnull + public static BindingMatcher tableFunctionExpression() { + return ofTypeOwning(TableFunctionExpression.class, CollectionMatcher.empty()); + } + @Nonnull public static BindingMatcher groupByExpression(@Nonnull final BindingMatcher downstream) { return ofTypeOwning(GroupByExpression.class, only(downstream)); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java index c74fa0ede8..8c835f70d3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/CardinalitiesProperty.java @@ -36,6 +36,7 @@ import com.apple.foundationdb.record.query.plan.cascades.expressions.GroupByExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.InsertExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.RecursiveUnionExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TableFunctionExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.TempTableInsertExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalDistinctExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.LogicalFilterExpression; @@ -75,6 +76,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; @@ -248,6 +250,12 @@ public Cardinalities visitRecordQueryInsertPlan(@Nonnull final RecordQueryInsert return fromChild(insertPlan); } + @Nonnull + @Override + public Cardinalities visitRecordQueryTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) { + return Cardinalities.unknownMaxCardinality(); + } + @Nonnull @Override public Cardinalities visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { @@ -559,6 +567,12 @@ public Cardinalities visitLogicalIntersectionExpression(@Nonnull final LogicalIn return intersectCardinalities(fromChildren(logicalIntersectionExpression)); } + @Nonnull + @Override + public Cardinalities visitTableFunctionExpression(@Nonnull final TableFunctionExpression element) { + return Cardinalities.unknownMaxCardinality(); + } + @Nonnull @Override public Cardinalities visitLogicalUniqueExpression(@Nonnull final LogicalUniqueExpression logicalUniqueExpression) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java index b2fe773d8c..f755dfe950 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java @@ -60,6 +60,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; @@ -336,6 +337,15 @@ public Derivations visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPl return new Derivations(resultValuesBuilder.build(), localValuesBuilder.build()); } + @Nonnull + @Override + public Derivations visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan tableFunctionPlan) { + final var collectionValue = tableFunctionPlan.getValue(); + final var elementType = collectionValue.getResultType(); + final var values = ImmutableList.of(new FirstOrDefaultValue(collectionValue, new ThrowsValue(elementType))); + return new Derivations(values, values); + } + @Nonnull @Override public Derivations visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java index ecb0a4a020..3a62e651cb 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DistinctRecordsProperty.java @@ -47,6 +47,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; @@ -214,6 +215,12 @@ public Boolean visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPlan) return distinctRecordsFromSingleChild(insertPlan); } + @Nonnull + @Override + public Boolean visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) { + return false; + } + @Nonnull @Override public Boolean visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java index e994a83b62..3e82785a1e 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/OrderingProperty.java @@ -59,6 +59,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; @@ -302,6 +303,12 @@ public Ordering visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPlan) return Ordering.empty(); } + @Nonnull + @Override + public Ordering visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) { + return Ordering.empty(); + } + @Nonnull @Override public Ordering visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java index f984611bb2..cd714cfb76 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/PrimaryKeyProperty.java @@ -48,6 +48,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableScanPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; @@ -216,6 +217,12 @@ public Optional> visitInsertPlan(@Nonnull final RecordQueryInsertPla return Optional.empty(); } + @Nonnull + @Override + public Optional> visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) { + return Optional.empty(); + } + @Nonnull @Override public Optional> visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java index ab2584f85a..8a82cd5b34 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/StoredRecordProperty.java @@ -45,6 +45,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryRecursiveUnionPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.TempTableInsertPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnKeyExpressionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryIntersectionOnValuesPlan; @@ -203,6 +204,12 @@ public Boolean visitInsertPlan(@Nonnull final RecordQueryInsertPlan element) { return true; } + @Nonnull + @Override + public Boolean visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan element) { + return false; + } + @Nonnull @Override public Boolean visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan element) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTableFunctionRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTableFunctionRule.java new file mode 100644 index 0000000000..7a8360dc07 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTableFunctionRule.java @@ -0,0 +1,52 @@ +/* + * ImplementTableFunctionRule.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.rules; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.query.plan.cascades.CascadesRule; +import com.apple.foundationdb.record.query.plan.cascades.CascadesRuleCall; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TableFunctionExpression; +import com.apple.foundationdb.record.query.plan.cascades.matching.structure.BindingMatcher; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; + +import javax.annotation.Nonnull; + +import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RelationalExpressionMatchers.tableFunctionExpression; + +/** + * A rule that implements an table function expression into a {@link RecordQueryTableFunctionPlan}. + */ +@API(API.Status.EXPERIMENTAL) +@SuppressWarnings("PMD.TooManyStaticImports") +public class ImplementTableFunctionRule extends CascadesRule { + private static final BindingMatcher root = + tableFunctionExpression(); + + public ImplementTableFunctionRule() { + super(root); + } + + @Override + public void onMatch(@Nonnull final CascadesRuleCall call) { + final var explodeExpression = call.get(root); + call.yieldExpression(new RecordQueryTableFunctionPlan(explodeExpression.getValue())); + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java index d73bad5895..59ef66aa03 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/typing/Type.java @@ -2771,6 +2771,7 @@ public Type getElementType() { * matches the current element type, the same instance is returned. */ @Nonnull + @SuppressWarnings("PMD.BrokenNullCheck") // I think PMD got confused or the null check conjunctions below. public Type.Array withElementType(@Nullable final Type elementType) { if (elementType == null && this.elementType == null) { return this; diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java index 27f0fb6083..2f58745bd3 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java @@ -37,6 +37,8 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.query.plan.cascades.BuiltInFunction; import com.apple.foundationdb.record.query.plan.cascades.BuiltInTableFunction; +import com.apple.foundationdb.record.query.plan.cascades.Column; +import com.apple.foundationdb.record.query.plan.cascades.SemanticException; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.typing.Typed; import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; @@ -54,16 +56,18 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executor; +import java.util.function.Function; import java.util.function.Supplier; /** * A Value that is able to return a range between 0 (inclusive) and a given number (inclusive). */ @SuppressWarnings("OptionalUsedAsFieldOrParameterType") -public class RangeValue extends AbstractValue implements StreamingValue { +public class RangeValue extends AbstractValue implements StreamingValue, CreatesDynamicTypesValue { private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Range-Value"); @Nonnull @@ -75,11 +79,21 @@ public class RangeValue extends AbstractValue implements StreamingValue { @Nonnull private final Optional step; + @Nonnull + private final Value currentRangeValue; + public RangeValue(@Nonnull final Value endExclusive, @Nonnull final Optional beginInclusive, @Nonnull final Optional step) { this.endExclusive = endExclusive; this.beginInclusive = beginInclusive; this.step = step; + currentRangeValue = RecordConstructorValue.ofColumns(ImmutableList.of(Column.of(Optional.of("ID"), LiteralValue.ofScalar(-1L)))); + } + + @Nonnull + @Override + public Type.Record getResultType() { + return (Type.Record)currentRangeValue.getResultType(); } @Nonnull @@ -91,7 +105,12 @@ public RecordCursor evalAsStream(@Nonnull final final long endExclusiveValue = (Long)Verify.verifyNotNull(endExclusive.eval(store, context)); final var beginInclusiveMaybe = beginInclusive.map(b -> (Long)Verify.verifyNotNull(b.eval(store, context))); final var stepMaybe = step.map(s -> (Long)Verify.verifyNotNull(s.eval(store, context))); - return new Cursor(store.getExecutor(), endExclusiveValue, beginInclusiveMaybe, stepMaybe, continuation); + return new Cursor(store.getExecutor(), endExclusiveValue, beginInclusiveMaybe, stepMaybe, rangeValueAsLong -> Objects.requireNonNull(currentRangeValue.replace(v -> { + if (v instanceof LiteralValue) { + return LiteralValue.ofScalar(rangeValueAsLong); + } + return v; + })).eval(store, context), continuation); } @Nonnull @@ -103,7 +122,7 @@ public ExplainTokensWithPrecedence explain(@Nonnull final Iterable Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { - throw new IllegalStateException("unable to eval an aggregation function with eval()"); + throw new IllegalStateException("unable to eval an streaming value with eval()"); } @Override @@ -144,16 +163,34 @@ protected Iterable computeChildren() { @Nonnull @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") // intentional. public Value withChildren(final Iterable newChildren) { final var newChildrenSize = Iterables.size(newChildren); Verify.verify(newChildrenSize <= 3 && newChildrenSize > 0); - final var end = Iterables.get(newChildren, 0); - final Optional begin = newChildrenSize > 1 ? Optional.of(Iterables.get(newChildren, 1)) : Optional.empty(); - final Optional step = newChildrenSize > 2 ? Optional.of(Iterables.get(newChildren, 2)) : Optional.empty(); - if (end != this.endExclusive || begin != this.beginInclusive || step != this.step) { - return new RangeValue(end, begin, step); + final var newEndExclusive = Iterables.get(newChildren, 0); + + if (newChildrenSize == 1 && newEndExclusive == this.endExclusive) { + return this; + } + + Optional newBeginInclusive = Optional.empty(); + if (newChildrenSize > 1) { + Verify.verify(beginInclusive.isPresent()); + newBeginInclusive = Optional.of(Iterables.get(newChildren, 1)); + if (newChildrenSize == 2 && newEndExclusive == this.endExclusive && newBeginInclusive.get() == beginInclusive.get()) { + return this; + } + } + + Optional newStep = Optional.empty(); + if (newChildrenSize > 2) { + Verify.verify(step.isPresent()); + newStep = Optional.of(Iterables.get(newChildren, 2)); + if (newEndExclusive == this.endExclusive && newBeginInclusive.get() == beginInclusive.get() && newStep.get() == step.get()) { + return this; + } } - return this; + return new RangeValue(newEndExclusive, newBeginInclusive, newStep); } @Nonnull @@ -183,16 +220,23 @@ public static class Cursor implements RecordCursor { private boolean closed = false; - public Cursor(@Nonnull final Executor executor, final long endExclusive, @Nonnull final Optional beginInclusive, - @Nonnull final Optional step, @Nullable final byte[] continuation) { - this(executor, endExclusive, step, continuation == null ? beginInclusive.orElse(0L) : Continuation.from(continuation).getNextPosition()); + @Nonnull + private final Function rangeValueCreator; + + Cursor(@Nonnull final Executor executor, final long endExclusive, @Nonnull final Optional beginInclusive, + @Nonnull final Optional step, @Nonnull final Function rangeValueCreator, @Nullable final byte[] continuation) { + this(executor, endExclusive, step, rangeValueCreator, + continuation == null ? beginInclusive.orElse(0L) : Continuation.from(continuation).getNextPosition()); } - private Cursor(@Nonnull final Executor executor, final long endExclusive, @Nonnull final Optional step, final long nextPosition) { + private Cursor(@Nonnull final Executor executor, final long endExclusive, @Nonnull final Optional step, + @Nonnull final Function rangeValueCreator, final long nextPosition) { + checkValidRange(nextPosition, endExclusive, step); this.executor = executor; this.endExclusive = endExclusive; this.step = step; this.nextPosition = nextPosition; + this.rangeValueCreator = rangeValueCreator; } @Nonnull @@ -206,8 +250,8 @@ public CompletableFuture> onNext() { public RecordCursorResult getNext() { RecordCursorResult nextResult; if (nextPosition < endExclusive) { - nextResult = RecordCursorResult.withNextValue(QueryResult.ofComputed(nextPosition), - new Continuation(endExclusive, nextPosition + step.orElse(1L), step)); + final var continuation = new Continuation(endExclusive, nextPosition + step.orElse(1L), step); + nextResult = RecordCursorResult.withNextValue(QueryResult.ofComputed(rangeValueCreator.apply(nextPosition)), continuation); nextPosition += step.orElse(1L); } else { nextResult = RecordCursorResult.exhausted(); @@ -237,6 +281,18 @@ public Executor getExecutor() { return executor; } + private static void checkValidRange(long position, long endExclusive, @Nonnull final Optional step) { + if (position < 0) { + throw new RecordCoreException("only non-negative position is allowed in range"); + } + if (endExclusive < 0) { + throw new RecordCoreException("only non-negative exclusive end is allowed in range"); + } + if (step.isPresent() && step.get() <= 0) { + throw new RecordCoreException("only positive step is allowed in range"); + } + } + private static class Continuation implements RecordCursorContinuation { private final long endExclusive; private final long nextPosition; @@ -254,8 +310,7 @@ public boolean isEnd() { // If a next value is returned as part of a cursor result, the continuation must not be an end continuation // (i.e., isEnd() must be false), per the contract of RecordCursorResult. This is the case even if the // cursor knows for certain that there is no more after that result, as in the ListCursor. - // Concretely, this means that we really need a > here, rather than >=. - return nextPosition > endExclusive; + return nextPosition >= endExclusive + step.orElse(1L); } public long getEndExclusive() { @@ -317,17 +372,44 @@ public static Continuation from(@Nonnull final byte[] unparsedContinuationBytes) public static class RangeFn extends BuiltInTableFunction { @Nonnull - private static StreamingValue encapsulateInternal(@Nonnull final BuiltInFunction builtInFunction, - @Nonnull final List arguments) { + private static StreamingValue encapsulateInternal(@Nonnull final List arguments) { Verify.verify(!arguments.isEmpty()); - final var endExclusive = PromoteValue.inject((Value)arguments.get(0), Type.primitiveType(Type.TypeCode.LONG)); - final Optional beginInclusive = arguments.size() > 1 ? Optional.of(PromoteValue.inject((Value)arguments.get(1), Type.primitiveType(Type.TypeCode.LONG))) : Optional.empty(); - final Optional step = arguments.size() > 2 ? Optional.of(PromoteValue.inject((Value)arguments.get(2), Type.primitiveType(Type.TypeCode.LONG))) : Optional.empty(); - return new RangeValue(endExclusive, beginInclusive, step); + final Value endExclusive; + final Optional beginInclusiveMaybe; + final Optional stepMaybe; + if (arguments.size() == 1) { + endExclusive = PromoteValue.inject((Value)arguments.get(0), Type.primitiveType(Type.TypeCode.LONG)); + checkValidBoundaryType(endExclusive); + beginInclusiveMaybe = Optional.empty(); + stepMaybe = Optional.empty(); + } else { + checkValidBoundaryType((Value)arguments.get(0)); + beginInclusiveMaybe = Optional.of(PromoteValue.inject((Value)arguments.get(0), Type.primitiveType(Type.TypeCode.LONG))); + checkValidBoundaryType((Value)arguments.get(1)); + endExclusive = PromoteValue.inject((Value)arguments.get(1), Type.primitiveType(Type.TypeCode.LONG)); + if (arguments.size() > 2) { + checkValidBoundaryType((Value)arguments.get(2)); + stepMaybe = Optional.of(PromoteValue.inject((Value)arguments.get(2), Type.primitiveType(Type.TypeCode.LONG))); + } else { + stepMaybe = Optional.empty(); + } + } + return new RangeValue(endExclusive, beginInclusiveMaybe, stepMaybe); } public RangeFn() { - super("range", ImmutableList.of(), new Type.Any(), RangeFn::encapsulateInternal); + super("range", ImmutableList.of(), new Type.Any(), (ignored, arguments) -> encapsulateInternal(arguments)); } } + + private static void checkValidBoundaryType(@Nonnull final Value value) { + final var type = value.getResultType(); + SemanticException.check(type.isPrimitive(), SemanticException.ErrorCode.INCOMPATIBLE_TYPE); + final var maxType = Type.maximumType(type, Type.primitiveType(Type.TypeCode.LONG)); + SemanticException.check(maxType != null, SemanticException.ErrorCode.INCOMPATIBLE_TYPE); + // currently, we do not support non-deterministic boundaries since we do not pre-compute them and store + // them in the continuation, but we can change enable this if required. + SemanticException.check(value.preOrderStream().filter(NondeterministicValue.class::isInstance).findAny().isEmpty(), + SemanticException.ErrorCode.UNSUPPORTED); + } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java index 7fae8e7b06..163a78c1ff 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java @@ -30,11 +30,14 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +/** + * a {@link Value} that returns a stream of results upon evaluation. + */ public interface StreamingValue extends Value { @Nonnull - RecordCursor evalAsStream(@Nonnull final FDBRecordStoreBase store, - @Nonnull final EvaluationContext context, - @Nullable final byte[] continuation, - @Nonnull final ExecuteProperties executeProperties); + RecordCursor evalAsStream(@Nonnull FDBRecordStoreBase store, + @Nonnull EvaluationContext context, + @Nullable byte[] continuation, + @Nonnull ExecuteProperties executeProperties); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java index 05de1220e5..c2eeb90ff9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java @@ -62,6 +62,7 @@ import com.apple.foundationdb.record.query.plan.plans.RecordQueryScoreForRankPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQuerySelectorPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryStreamingAggregationPlan; +import com.apple.foundationdb.record.query.plan.plans.RecordQueryTableFunctionPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTextIndexPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryTypeFilterPlan; import com.apple.foundationdb.record.query.plan.plans.RecordQueryUnionOnKeyExpressionPlan; @@ -414,6 +415,13 @@ public ExplainTokens visitInsertPlan(@Nonnull final RecordQueryInsertPlan insert .addIdentifier(insertPlan.getTargetRecordType()); } + @Nonnull + @Override + public ExplainTokens visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan tableFunctionPlan) { + return addKeyword("TFunc").addWhitespace() + .addNested(tableFunctionPlan.getValue().explain().getExplainTokens()); + } + @Nonnull @Override public ExplainTokens visitTempTableInsertPlan(@Nonnull final TempTableInsertPlan tempTableInsertPlan) { diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java new file mode 100644 index 0000000000..be3807b8a9 --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java @@ -0,0 +1,280 @@ +/* + * RecordQueryTableValuedPlan.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.plans; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ExecuteProperties; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.planprotos.PRecordQueryPlan; +import com.apple.foundationdb.record.planprotos.PRecordQueryTableFunctionPlan; +import com.apple.foundationdb.record.provider.common.StoreTimer; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.AvailableFields; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; +import com.apple.foundationdb.record.query.plan.cascades.Memoizer; +import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.explain.Attribute; +import com.apple.foundationdb.record.query.plan.cascades.explain.NodeInfo; +import com.apple.foundationdb.record.query.plan.cascades.explain.PlannerGraph; +import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.cascades.values.QueriedValue; +import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue; +import com.apple.foundationdb.record.query.plan.cascades.values.Value; +import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; +import com.apple.foundationdb.record.query.plan.explain.ExplainPlanVisitor; +import com.google.auto.service.AutoService; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.Set; + +/** + * A query plan that delegates its execution to a table-valued {@link StreamingValue}. + */ +@API(API.Status.INTERNAL) +public class RecordQueryTableFunctionPlan implements RecordQueryPlanWithNoChildren { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Table-Function-Plan"); + + @Nonnull + private final StreamingValue value; + + public RecordQueryTableFunctionPlan(@Nonnull final StreamingValue collectionValue) { + this.value = collectionValue; + } + + @Nonnull + @Override + public RecordCursor executePlan(@Nonnull final FDBRecordStoreBase store, + @Nonnull final EvaluationContext context, + @Nullable final byte[] continuation, + @Nonnull final ExecuteProperties executeProperties) { + return value.evalAsStream(store, context, continuation, executeProperties); + } + + @Nonnull + @Override + public Set getCorrelatedTo() { + return value.getCorrelatedTo(); + } + + @Nonnull + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public RecordQueryTableFunctionPlan translateCorrelations(@Nonnull final TranslationMap translationMap, + final boolean shouldSimplifyValues, + @Nonnull final List translatedQuantifiers) { + final Value translatedValue = value.translateCorrelations(translationMap, shouldSimplifyValues); + if (translatedValue != value) { + return new RecordQueryTableFunctionPlan((StreamingValue)translatedValue); + } + return this; + } + + @Override + public boolean isReverse() { + return false; + } + + @Override + public RecordQueryTableFunctionPlan strictlySorted(@Nonnull final Memoizer memoizer) { + return this; + } + + @Override + public boolean hasRecordScan() { + return false; + } + + @Override + public boolean hasFullRecordScan() { + return false; + } + + @Override + public boolean hasIndexScan(@Nonnull final String indexName) { + return false; + } + + @Nonnull + @Override + public Set getUsedIndexes() { + return ImmutableSet.of(); + } + + @Override + public boolean hasLoadBykeys() { + return false; + } + + @Nonnull + @Override + public AvailableFields getAvailableFields() { + return AvailableFields.NO_FIELDS; + } + + @Nonnull + @Override + public Value getResultValue() { + return new QueriedValue(value.getResultType()); + } + + @Nonnull + public StreamingValue getValue() { + return value; + } + + @Nonnull + @Override + public Set getDynamicTypes() { + return value.getDynamicTypes(); + } + + + @Nonnull + @Override + public String toString() { + return ExplainPlanVisitor.toStringForDebugging(this); + } + + @Override + @SuppressWarnings("PMD.CompareObjectsWithEquals") + public boolean equalsWithoutChildren(@Nonnull final RelationalExpression otherExpression, + @Nonnull final AliasMap equivalencesMap) { + if (this == otherExpression) { + return true; + } + if (getClass() != otherExpression.getClass()) { + return false; + } + final var otherTableFunctionPlan = (RecordQueryTableFunctionPlan)otherExpression; + + return value.semanticEquals(otherTableFunctionPlan.value, equivalencesMap) && + semanticEqualsForResults(otherExpression, equivalencesMap); + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @Override + public boolean equals(final Object other) { + return structuralEquals(other); + } + + @Override + public int hashCode() { + return structuralHashCode(); + } + + @Override + public int hashCodeWithoutChildren() { + return Objects.hash(getResultValue()); + } + + @Override + public void logPlanStructure(StoreTimer timer) { + // nothing to increment + } + + @Override + public int getComplexity() { + return 1; + } + + @Override + public int planHash(@Nonnull final PlanHashMode mode) { + switch (mode.getKind()) { + case LEGACY: + case FOR_CONTINUATION: + return PlanHashable.objectsPlanHash(mode, BASE_HASH, getResultValue()); + default: + throw new UnsupportedOperationException("Hash kind " + mode.getKind() + " is not supported"); + } + } + + @Nonnull + @Override + public PlannerGraph rewritePlannerGraph(@Nonnull final List childGraphs) { + return PlannerGraph.fromNodeAndChildGraphs( + new PlannerGraph.OperatorNodeWithInfo(this, + NodeInfo.VALUE_COMPUTATION_OPERATOR, + ImmutableList.of("TFUNC {{expr}}"), + ImmutableMap.of("expr", Attribute.gml(value.toString()))), + childGraphs); + } + + @Nonnull + @Override + public PRecordQueryTableFunctionPlan toProto(@Nonnull final PlanSerializationContext serializationContext) { + return PRecordQueryTableFunctionPlan.newBuilder() + .setValue(value.toValueProto(serializationContext)) + .build(); + } + + @Nonnull + @Override + public PRecordQueryPlan toRecordQueryPlanProto(@Nonnull final PlanSerializationContext serializationContext) { + return PRecordQueryPlan.newBuilder().setTableFunctionPlan(toProto(serializationContext)).build(); + } + + @Nonnull + public static RecordQueryTableFunctionPlan fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PRecordQueryTableFunctionPlan recordQueryTableFunctionPlanProto) { + final var value = Value.fromValueProto(serializationContext, + Objects.requireNonNull(recordQueryTableFunctionPlanProto.getValue())); + if (!(value instanceof StreamingValue)) { + throw new RecordCoreException("invalid value, expecting streaming value").addLogInfo(LogMessageKeys.VALUE, value); + } + return new RecordQueryTableFunctionPlan((StreamingValue)value); + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PRecordQueryTableFunctionPlan.class; + } + + @Nonnull + @Override + public RecordQueryTableFunctionPlan fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PRecordQueryTableFunctionPlan message) { + return RecordQueryTableFunctionPlan.fromProto(serializationContext, message); + } + } +} + diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 003ce212fc..8badb0dabd 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -1227,6 +1227,7 @@ message PRecordQueryPlan { PTempTableScanPlan temp_table_scan_plan = 34; PTempTableInsertPlan temp_table_insert_plan = 35; PRecursiveUnionQueryPlan recursive_union_query_plan = 36; + PRecordQueryTableFunctionPlan table_function_plan = 37; } } @@ -1819,3 +1820,10 @@ message PRecursiveUnionQueryPlan { optional string initialTempTableAlias = 3; optional string recursiveTempTableAlias = 4; } + +// +// PRecordQueryTableFunctionPlan +// +message PRecordQueryTableFunctionPlan { + optional PValue value = 1; +} diff --git a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 index 72879bb762..54d6109f20 100644 --- a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 @@ -166,7 +166,7 @@ PARTITION: 'PARTITION'; PRIMARY: 'PRIMARY'; PROCEDURE: 'PROCEDURE'; PURGE: 'PURGE'; -RANGE: 'RANGE'; +// RANGE: 'RANGE'; READ: 'READ'; READS: 'READS'; RECURSIVE: 'RECURSIVE'; diff --git a/fdb-relational-core/src/main/antlr/RelationalParser.g4 b/fdb-relational-core/src/main/antlr/RelationalParser.g4 index 8b6f298d99..6d1190c172 100644 --- a/fdb-relational-core/src/main/antlr/RelationalParser.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalParser.g4 @@ -227,7 +227,11 @@ namedQuery ; tableFunction - : functionNameBase '(' functionArgs? ')' inlineTableDefinition? + : tableFunctionName '(' functionArgs? ')' inlineTableDefinition? + ; + +tableFunctionName + : fullId ; continuation @@ -287,9 +291,9 @@ tableSource // done tableSourceItem // done : tableName (AS? alias=uid)? (indexHint (',' indexHint)* )? #atomTableItem // done - | query AS? alias=uid #subqueryTableItem // done + | '(' query ')' AS? alias=uid #subqueryTableItem // done | VALUES recordConstructorForInlineTable (',' recordConstructorForInlineTable )* inlineTableDefinition? #inlineTableItem - | tableFunction #tableValuedFunction + | tableFunction (AS? alias=uid)? #tableValuedFunction ; indexHint @@ -1146,7 +1150,7 @@ keywordsCanBeId | TEXT | TEMPORARY | TEMPTABLE | THAN | TRADITIONAL | TRANSACTION | TRANSACTIONAL | TRIGGERS | TRUNCATE | UNDEFINED | UNDOFILE | UNDO_BUFFER_SIZE | UNINSTALL | UNKNOWN | UNTIL | UPGRADE | USA | USER | USE_FRM | USER_RESOURCES - | VALIDATION | VALUE | VALUES | VAR_POP | VAR_SAMP | VARIABLES | VARIANCE | VERSION_TOKEN_ADMIN | VIEW | WAIT | WARNINGS | WITHOUT + | VALIDATION | VALUE | VAR_POP | VAR_SAMP | VARIABLES | VARIANCE | VERSION_TOKEN_ADMIN | VIEW | WAIT | WARNINGS | WITHOUT | WRAPPER | X509 | XA | XA_RECOVER_ADMIN | XML ; diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java index 340b04a41e..bd170172d6 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/LogicalOperator.java @@ -56,7 +56,6 @@ import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; import com.apple.foundationdb.relational.util.Assert; -import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Iterables; @@ -283,24 +282,6 @@ public static Expressions convertToExpressions(@Nonnull Quantifier quantifier) { return Expressions.of(attributesBuilder.build()); } - @Nonnull - public static Expressions convertToExpressions(@Nonnull final Quantifier quantifier, @Nonnull final Type type) { - Verify.verify(type.isRecord()); - final var recordType = (Type.Record)type; - final ImmutableList.Builder attributesBuilder = ImmutableList.builder(); - int colCount = 0; - final var columns = quantifier.getFlowedColumns(); - for (final var column : columns) { - final var field = recordType.getFields().get(colCount); - final var attributeName = field.getFieldNameOptional().map(Identifier::of); - final var attributeType = DataTypeUtils.toRelationalType(field.getFieldType()); - final var attributeExpression = FieldValue.ofOrdinalNumber(quantifier.getFlowedObjectValue(), colCount); - attributesBuilder.add(new Expression(attributeName, attributeType, attributeExpression)); - colCount++; - } - return Expressions.of(attributesBuilder.build()); - } - @Nonnull public static LogicalOperator generateSelect(@Nonnull Expressions output, @Nonnull LogicalOperators logicalOperators, diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java index 19ac822c26..6a7824a029 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/SemanticAnalyzer.java @@ -24,6 +24,7 @@ import com.apple.foundationdb.record.query.plan.cascades.AccessHint; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.BuiltInTableFunction; import com.apple.foundationdb.record.query.plan.cascades.Correlated; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; import com.apple.foundationdb.record.query.plan.cascades.IndexAccessHint; @@ -732,15 +733,21 @@ public static void validateContinuation(@Nonnull Expression continuation) { * @param functionName The function name. * @param flattenSingleItemRecords {@code true} if single-item records should be (recursively) replaced with their * content, otherwise {@code false}. + * @param isTableValued {@code true} if the function is expected to be a table-valued function, otherwise {@code false}. * @param arguments The function arguments. * @return A resolved SQL function {@code Expression}. */ @Nonnull - public Expression resolveFunction(@Nonnull final String functionName, boolean flattenSingleItemRecords, + public Expression resolveFunction(@Nonnull final String functionName, + boolean flattenSingleItemRecords, + boolean isTableValued, @Nonnull final Expression... arguments) { Assert.thatUnchecked(functionCatalog.containsFunction(functionName), ErrorCode.UNSUPPORTED_QUERY, () -> String.format(Locale.ROOT, "Unsupported operator %s", functionName)); final var builtInFunction = functionCatalog.lookUpFunction(functionName, arguments); + if (isTableValued) { + Assert.thatUnchecked(builtInFunction instanceof BuiltInTableFunction, functionName + " is not a table-valued function"); + } List argumentList = new ArrayList<>(); argumentList.addAll(List.of(arguments)); if (BITMAP_SCALAR_FUNCTIONS.contains(functionName.toLowerCase(Locale.ROOT))) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java index 31ee33c108..a983a33768 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/functions/SqlFunctionCatalogImpl.java @@ -111,6 +111,7 @@ private static ImmutableMap FunctionCatalog.resolve("coalesce", argumentsCount).orElseThrow()) .put("is null", argumentsCount -> FunctionCatalog.resolve("isNull", argumentsCount).orElseThrow()) .put("is not null", argumentsCount -> FunctionCatalog.resolve("notNull", argumentsCount).orElseThrow()) + .put("range", argumentsCount -> FunctionCatalog.resolve("range", argumentsCount).orElseThrow()) .put("__pattern_for_like", argumentsCount -> FunctionCatalog.resolve("patternForLike", argumentsCount).orElseThrow()) .put("__internal_array", argumentsCount -> FunctionCatalog.resolve("array", argumentsCount).orElseThrow()) .build(); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java index 3bfc28fd2d..a1d3e50dd4 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/BaseVisitor.java @@ -23,7 +23,6 @@ import com.apple.foundationdb.annotation.API; import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; -import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.ddl.DdlQueryFactory; import com.apple.foundationdb.relational.api.ddl.MetadataOperationsFactory; @@ -211,12 +210,17 @@ protected String normalizeString(@Nonnull final String value) { @Nonnull public Expression resolveFunction(@Nonnull String functionName, @Nonnull Expression... arguments) { - return getSemanticAnalyzer().resolveFunction(functionName, true, arguments); + return getSemanticAnalyzer().resolveFunction(functionName, true, false, arguments); } @Nonnull public Expression resolveFunction(@Nonnull String functionName, boolean flattenSingleItemRecords, @Nonnull Expression... arguments) { - return getSemanticAnalyzer().resolveFunction(functionName, flattenSingleItemRecords, arguments); + return getSemanticAnalyzer().resolveFunction(functionName, flattenSingleItemRecords, false, arguments); + } + + @Nonnull + public Expression resolveTableValuedFunction(@Nonnull String functionName, @Nonnull Expression... arguments) { + return getSemanticAnalyzer().resolveFunction(functionName, true, true, arguments); } @Override @@ -473,6 +477,11 @@ public Expression visitTableFunction(@Nonnull final RelationalParser.TableFuncti return expressionVisitor.visitTableFunction(ctx); } + @Override + public Identifier visitTableFunctionName(final RelationalParser.TableFunctionNameContext ctx) { + return identifierVisitor.visitTableFunctionName(ctx); + } + @Nonnull @Override public Expression visitContinuation(RelationalParser.ContinuationContext ctx) { diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java index f9e916cdee..bb67e6f20a 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/DelegatingVisitor.java @@ -312,6 +312,11 @@ public Expression visitTableFunction(final RelationalParser.TableFunctionContext return getDelegate().visitTableFunction(ctx); } + @Override + public Identifier visitTableFunctionName(final RelationalParser.TableFunctionNameContext ctx) { + return getDelegate().visitTableFunctionName(ctx); + } + @Nonnull @Override public Expression visitContinuation(@Nonnull RelationalParser.ContinuationContext ctx) { @@ -404,12 +409,12 @@ public LogicalOperator visitSubqueryTableItem(@Nonnull RelationalParser.Subquery @Nonnull @Override - public LogicalOperator visitInlineTableItem(final RelationalParser.InlineTableItemContext ctx) { + public LogicalOperator visitInlineTableItem(@Nonnull final RelationalParser.InlineTableItemContext ctx) { return getDelegate().visitInlineTableItem(ctx); } @Override - public LogicalOperator visitTableValuedFunction(final RelationalParser.TableValuedFunctionContext ctx) { + public LogicalOperator visitTableValuedFunction(@Nonnull final RelationalParser.TableValuedFunctionContext ctx) { return getDelegate().visitTableValuedFunction(ctx); } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java index 120eb0e3e7..a005a3d87b 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/ExpressionVisitor.java @@ -88,8 +88,11 @@ public static ExpressionVisitor of(@Nonnull BaseVisitor baseVisitor) { } @Override - public Expression visitTableFunction(final RelationalParser.TableFunctionContext ctx) { - return super.visitTableFunction(ctx); + public Expression visitTableFunction(@Nonnull RelationalParser.TableFunctionContext ctx) { + final var functionName = visitTableFunctionName(ctx.tableFunctionName()).toString(); + return ctx.functionArgs() == null + ? getDelegate().resolveTableValuedFunction(functionName) + : getDelegate().resolveTableValuedFunction(functionName, visitFunctionArgs(ctx.functionArgs()).asList().toArray(new Expression[0])); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/IdentifierVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/IdentifierVisitor.java index 73b0a44acb..b400e64892 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/IdentifierVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/IdentifierVisitor.java @@ -117,4 +117,9 @@ public Identifier visitCollationName(@Nonnull RelationalParser.CollationNameCont Assert.failUnchecked(ErrorCode.UNSUPPORTED_QUERY, "setting collation is not supported"); return null; } + + @Override + public Identifier visitTableFunctionName(final RelationalParser.TableFunctionNameContext ctx) { + return visitFullId(ctx.fullId()); + } } diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java index ad8cfd165e..013005826f 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/QueryVisitor.java @@ -28,15 +28,16 @@ import com.apple.foundationdb.record.query.plan.cascades.expressions.DeleteExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.ExplodeExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.RecursiveUnionExpression; +import com.apple.foundationdb.record.query.plan.cascades.expressions.TableFunctionExpression; import com.apple.foundationdb.record.query.plan.cascades.expressions.UpdateExpression; import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; import com.apple.foundationdb.record.query.plan.cascades.typing.Type; import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.query.plan.cascades.values.LiteralValue; +import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue; import com.apple.foundationdb.record.query.plan.cascades.values.Value; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.exceptions.ErrorCode; -import com.apple.foundationdb.relational.api.metadata.Table; import com.apple.foundationdb.relational.generated.RelationalLexer; import com.apple.foundationdb.relational.generated.RelationalParser; import com.apple.foundationdb.relational.recordlayer.metadata.RecordLayerTable; @@ -313,9 +314,8 @@ public LogicalOperator visitTableSourceBase(@Nonnull RelationalParser.TableSourc @Override public LogicalOperator visitAtomTableItem(@Nonnull RelationalParser.AtomTableItemContext atomTableItemContext) { final var tableIdentifier = Assert.castUnchecked(atomTableItemContext.tableName().accept(this), Identifier.class); - final var tableAlias = Optional.of(atomTableItemContext.alias == null ? - Assert.castUnchecked(atomTableItemContext.tableName().accept(this), Identifier.class) : - Assert.castUnchecked(atomTableItemContext.alias.accept(this), Identifier.class)); + final var tableAlias = Optional.of(atomTableItemContext.alias == null ? visitTableName(atomTableItemContext.tableName()) + : visitUid(atomTableItemContext.alias)); final var requestedIndexes = atomTableItemContext.indexHint() .stream().flatMap(indexHint -> visitIndexHint(indexHint).stream()).collect(ImmutableSet.toImmutableSet()); return LogicalOperator.generateAccess(tableIdentifier, tableAlias, requestedIndexes, getDelegate().getSemanticAnalyzer(), @@ -367,7 +367,14 @@ public LogicalOperator visitInlineTableItem(@Nonnull RelationalParser.InlineTabl @Override public LogicalOperator visitTableValuedFunction(@Nonnull RelationalParser.TableValuedFunctionContext tableValuedFunctionContext) { - return null; + final var expression = visitTableFunction(tableValuedFunctionContext.tableFunction()); + final var underlyingValue = expression.getUnderlying(); + final var explodeExpression = new TableFunctionExpression(Assert.castUnchecked(underlyingValue, StreamingValue.class)); + final var resultingQuantifier = Quantifier.forEach(Reference.of(explodeExpression)); + final var output = Expressions.of(LogicalOperator.convertToExpressions(resultingQuantifier)); + final var aliasMaybe = Optional.ofNullable(tableValuedFunctionContext.uid() == null ? null : visitUid(tableValuedFunctionContext.uid())); + return aliasMaybe.map(alias -> LogicalOperator.newNamedOperator(alias, output, resultingQuantifier)) + .orElse(LogicalOperator.newUnnamedOperator(output, resultingQuantifier)); } @Nonnull diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java index 03796eb89a..16d2eef5b1 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/query/visitors/TypedVisitor.java @@ -21,7 +21,6 @@ package com.apple.foundationdb.relational.recordlayer.query.visitors; import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; -import com.apple.foundationdb.record.query.plan.cascades.values.FieldValue; import com.apple.foundationdb.record.util.pair.NonnullPair; import com.apple.foundationdb.relational.api.metadata.DataType; import com.apple.foundationdb.relational.generated.RelationalParser; @@ -218,6 +217,9 @@ public interface TypedVisitor extends RelationalParserVisitor { @Override Expression visitTableFunction(@Nonnull RelationalParser.TableFunctionContext ctx); + @Override + Identifier visitTableFunctionName(RelationalParser.TableFunctionNameContext ctx); + @Nonnull @Override Expression visitContinuation(RelationalParser.ContinuationContext ctx); @@ -280,7 +282,7 @@ public interface TypedVisitor extends RelationalParserVisitor { @Nonnull @Override - LogicalOperator visitInlineTableItem(final RelationalParser.InlineTableItemContext ctx); + LogicalOperator visitInlineTableItem(@Nonnull RelationalParser.InlineTableItemContext ctx); @Override LogicalOperator visitTableValuedFunction(@Nonnull RelationalParser.TableValuedFunctionContext ctx); diff --git a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/TypeUtils.java b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/TypeUtils.java index edf78ae7da..45f297724d 100644 --- a/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/TypeUtils.java +++ b/fdb-relational-core/src/main/java/com/apple/foundationdb/relational/recordlayer/util/TypeUtils.java @@ -1,3 +1,24 @@ +/* + * TypeUtils.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2021-2025 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package com.apple.foundationdb.relational.recordlayer.util; import com.apple.foundationdb.record.query.plan.cascades.predicates.CompatibleTypeEvolutionPredicate; @@ -21,6 +42,8 @@ public static Type setFieldNames(@Nonnull final Type input, } @Nonnull + // PMD incorrectly thinks that comparing array sizes it deemed to be object reference comparison requiring equals() instead. + @SuppressWarnings("PMD.CompareObjectsWithEquals") private static Type setFieldNamesInternal(@Nonnull final Type input, @Nonnull final CompatibleTypeEvolutionPredicate.FieldAccessTrieNode trie) { if (input.isPrimitive()) { diff --git a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/YamlTestExtension.java b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/YamlTestExtension.java index e502281278..58f1307585 100644 --- a/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/YamlTestExtension.java +++ b/yaml-tests/src/main/java/com/apple/foundationdb/relational/yamltests/YamlTestExtension.java @@ -31,7 +31,6 @@ import com.apple.foundationdb.relational.yamltests.configs.ShowPlanOnDiff; import com.apple.foundationdb.relational.yamltests.configs.YamlTestConfig; import com.apple.foundationdb.relational.yamltests.server.ExternalServer; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; diff --git a/yaml-tests/src/test/resources/table-functions.yamsql b/yaml-tests/src/test/resources/table-functions.yamsql index 7e2d78056e..ae6e8772d3 100644 --- a/yaml-tests/src/test/resources/table-functions.yamsql +++ b/yaml-tests/src/test/resources/table-functions.yamsql @@ -20,11 +20,23 @@ options: supported_version: !current_version --- +schema_template: + create table t1(id bigint, col1 string, primary key(id)) +--- +setup: + steps: + - query: INSERT INTO T1 VALUES + (1, 'a'), + (2, 'b'), + (3, 'c') +--- test_block: - connect: "jdbc:embed:/__SYS?schema=CATALOG" name: table-functions preset: single_repetition_ordered tests: + - + - query: select * from values (42) + - result: [{42}] - - query: select * from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) - explain: "EXPLODE array((@c6 AS B, @c8 AS C, (@c11 AS X, promote(@c13 AS DOUBLE) AS Y, @c15 AS Z) AS W), (@c20 AS B, @c22 AS C, (@c25 AS X, @c27 AS Y, @c29 AS Z) AS W))" @@ -64,6 +76,52 @@ test_block: - explain: "EXPLODE array((@c20 AS B, @c22 AS C, (@c25 AS X, promote(@c27 AS DOUBLE) AS Y, @c29 AS Z) AS W), (@c34 AS B, @c36 AS C, (@c39 AS X, @c41 AS Y, @c43 AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X) | FILTER _.B LESS_THAN @c68" - result: [{B: 1, Q: 2.0, X: 3}] - - - query: select * from values() - - error: "42601" + - query: select * from range(1, 4) + - result: [{ID: 1}, {ID: 2}, {ID: 3}] + - + - query: select * from range(-1) + - error: XXXXX + - + - query: select * from range(-1, 4) + - error: XXXXX + - + - query: select * from range(1, 4, -1) + - error: XXXXX + - + - query: select * from range(1, 4, 0) + - error: XXXXX + - + - query: select * from range(0, 12, 5) + - result: [{ID: 0}, {ID: 5}, {ID: 10}] + - + - query: select * from range(0, 11, 5) + - result: [{ID: 0}, {ID: 5}, {ID: 10}] + - + - query: select * from range(6 - 6, 14 + 6 + 1, 20 - 10) + - result: [{ID: 0}, {ID: 10}, {ID: 20}] + - + - query: select ID as X from range(3) as Y + - result: [{X: 0}, {X: 1}, {X: 2}] + - + - query: select X.ID as A, Y.ID as B from range(3) as X, range(4) as Y + - result: [{A: 0, B: 0}, + {A: 0, B: 1}, + {A: 0, B: 2}, + {A: 0, B: 3}, + {A: 1, B: 0}, + {A: 1, B: 1}, + {A: 1, B: 2}, + {A: 1, B: 3}, + {A: 2, B: 0}, + {A: 2, B: 1}, + {A: 2, B: 2}, + {A: 2, B: 3}] + - + - query: select a.id as x, a.col1 as y, b.id as z from t1 as a, range(a.id) as b + - result: [{X: 1, Y: 'a', Z: 0}, + {X: 2, Y: 'b', Z: 0}, + {X: 2, Y: 'b', Z: 1}, + {X: 3, Y: 'c', Z: 0}, + {X: 3, Y: 'c', Z: 1}, + {X: 3, Y: 'c', Z: 2}] ... From 4cd375f95f088cafb8ac9295ff24281a5f84e980 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Fri, 28 Mar 2025 15:25:40 +0100 Subject: [PATCH 12/16] refactoring. --- .../plan/cascades/values/RangeValue.java | 31 +++---- .../src/main/proto/record_cursor.proto | 2 - .../src/test/java/YamlIntegrationTests.java | 1 - .../resources/table-functions.metrics.binpb | Bin 9650 -> 13845 bytes .../resources/table-functions.metrics.yaml | 87 ++++++++++++------ .../src/test/resources/table-functions.yamsql | 5 + 6 files changed, 75 insertions(+), 51 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java index 2f58745bd3..0a375550fc 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java @@ -64,7 +64,13 @@ import java.util.function.Supplier; /** - * A Value that is able to return a range between 0 (inclusive) and a given number (inclusive). + * A {@link StreamingValue} that is able to return a range that is defined as the following: + *
    + *
  • an optional inclusive start (0L by default).
    • + *
  • an exclusive end.
    • + *
  • an optional step (1L by default).
    • + *
+ * For more information, see relational SQL {@code range} table-valued function. */ @SuppressWarnings("OptionalUsedAsFieldOrParameterType") public class RangeValue extends AbstractValue implements StreamingValue, CreatesDynamicTypesValue { @@ -226,7 +232,7 @@ public static class Cursor implements RecordCursor { Cursor(@Nonnull final Executor executor, final long endExclusive, @Nonnull final Optional beginInclusive, @Nonnull final Optional step, @Nonnull final Function rangeValueCreator, @Nullable final byte[] continuation) { this(executor, endExclusive, step, rangeValueCreator, - continuation == null ? beginInclusive.orElse(0L) : Continuation.from(continuation).getNextPosition()); + continuation == null ? beginInclusive.orElse(0L) : Continuation.from(continuation, endExclusive, step).getNextPosition()); } private Cursor(@Nonnull final Executor executor, final long endExclusive, @Nonnull final Optional step, @@ -313,28 +319,17 @@ public boolean isEnd() { return nextPosition >= endExclusive + step.orElse(1L); } - public long getEndExclusive() { - return endExclusive; - } - public long getNextPosition() { return nextPosition; } - @Nonnull - public Optional getStep() { - return step; - } - @Nonnull @Override public ByteString toByteString() { if (isEnd()) { return ByteString.EMPTY; } - final var protoBuilder = RecordCursorProto.RangeCursorContinuation.newBuilder() - .setEndExclusive(endExclusive).setNextPosition(nextPosition); - step.ifPresent(protoBuilder::setStep); + final var protoBuilder = RecordCursorProto.RangeCursorContinuation.newBuilder().setNextPosition(nextPosition); return protoBuilder.build().toByteString(); } @@ -345,18 +340,16 @@ public byte[] toBytes() { } @Nonnull - public static Continuation from(@Nonnull final RecordCursorProto.RangeCursorContinuation message) { - final var endExclusive = message.getEndExclusive(); + public static Continuation from(@Nonnull final RecordCursorProto.RangeCursorContinuation message, long endExclusive, @Nonnull final Optional step) { final var nextPosition = message.getNextPosition(); - final Optional step = message.hasStep() ? Optional.of(message.getStep()) : Optional.empty(); return new Continuation(endExclusive, nextPosition, step); } @Nonnull - public static Continuation from(@Nonnull final byte[] unparsedContinuationBytes) { + public static Continuation from(@Nonnull final byte[] unparsedContinuationBytes, long endExclusive, @Nonnull final Optional step) { try { final var parsed = RecordCursorProto.RangeCursorContinuation.parseFrom(unparsedContinuationBytes); - return from(parsed); + return from(parsed, endExclusive, step); } catch (InvalidProtocolBufferException ex) { throw new RecordCoreException("invalid continuation", ex) .addLogInfo(LogMessageKeys.RAW_BYTES, ByteArrayUtil2.loggable(unparsedContinuationBytes)); diff --git a/fdb-record-layer-core/src/main/proto/record_cursor.proto b/fdb-record-layer-core/src/main/proto/record_cursor.proto index 8807f3d7d4..4cb306a829 100644 --- a/fdb-record-layer-core/src/main/proto/record_cursor.proto +++ b/fdb-record-layer-core/src/main/proto/record_cursor.proto @@ -134,7 +134,5 @@ message RecursiveCursorContinuation { } message RangeCursorContinuation { - optional int64 endExclusive = 1; optional int64 nextPosition = 2; - optional int64 step = 3; } \ No newline at end of file diff --git a/yaml-tests/src/test/java/YamlIntegrationTests.java b/yaml-tests/src/test/java/YamlIntegrationTests.java index a2469f521d..60aa81c8bb 100644 --- a/yaml-tests/src/test/java/YamlIntegrationTests.java +++ b/yaml-tests/src/test/java/YamlIntegrationTests.java @@ -255,7 +255,6 @@ public void enumTest(YamlTest.Runner runner) throws Exception { } @TestTemplate - @MaintainYamlTestConfig(YamlTestConfigFilters.CORRECT_EXPLAIN_AND_METRICS) public void tableFunctionsTest(YamlTest.Runner runner) throws Exception { runner.runYamsql("table-functions.yamsql"); } diff --git a/yaml-tests/src/test/resources/table-functions.metrics.binpb b/yaml-tests/src/test/resources/table-functions.metrics.binpb index 2ada5ca667cf51506bf8f416f9c5c19e4834c1f6..d2cd781e7cf311d39c479770041073b17ea09513 100644 GIT binary patch delta 1734 zcmb7EYe-vH9A6TzHAih?R+DJm@?V&eutbu0XM-A9j5_RMDjFfymU5F^vx`^L+}w^i zC&9I2Wt3reSW8)H>xO$j*uu(aN52U3Lt*O&_h77Sjxmrz2ZiqSVcq|l8*P^E(Lj=W z{^$J9`Mu8l@UfRf%`%;3a+YO=SgN{>KYp~Tv07l|;}(a`bL&W5-dNeCGrn?So>=Q) z_4gjE-gwTO->2_qjK8eErNQcrukOX_xqLm{r*Ft#-p(?{uYWqJ%=~=GgfpiXNZzm` z#h8{qJEgTg0LPa6jtbAp+r-b%x*g{HnTBzu?)76UI-Gkg|E2~f*S@?zPi${gTE5j0 zKQYK$T;$yoho%##tVa1)y9Rpo`l`S2-YO zUu-BG9Ed=c7x{#Y_<_qp3?`)Xl*+Hosg%Y{e^jDKy~fL{DfH*;wHm~<`1v+}1&U-V z5ec4#!MflOI1aSpb|^~B*?6fEb-St^=%s@@kLruyV)TRRQ)Ob3;xw*yTTkaM)R0&? z^1iFANC4e7H?qnd#P4S(Z!(KrOgog`bpCU>dj6)(Vrlo*g;*)mB$}=J#unW^q z2-%Ih0We4AsFA*?eK5zh3Q5JaCtbj`VwYDL%(x(~)}Ja^C~wrcqIpW!hJSI>I|kBF zby#CudhbM&)eHF-{MRMU*?;KO8;d z0{m%7$(?l3^33gb<$`_EFeC|6oHPR?a$4e2JY6F;lf);|QWE5ugM79F#-rZ-*tx4| zLJ&p8ALU>5OB7O%%W?4#k8xn@x2~ZJ3U4(4mS)TgWsz&#T3TG#rbNL9DJNAotu!yWBr`v+Sjczs0|~K-x&m(MY5Do;nmP&^h6Xwc zmIiu8ItuDZiACy~3W>!Ejv7un3eGwTE}B9sS-2!P>;!IHohBk7uCU|NITj6OgQe?? zESMb_gaRkdQ#N!+HZ)Lh3%48$g}U|$&OC>ZMjy#kZ~k|xF=!o(m}Q>c-Z zOC0EdrWuPFfgbq1NfPM(&kH!g?ysFVPb@1QsMH}D?7n!QDTzfziIpIK0NFqiXa$fo z)dV>|-U#AFBd~uV`izXgGDsSYOhHq&_|8LDI3+|CF8(M43a)wTV!;B8 z-J^_?FS0F|Y|AgX`6s6X)8s$QeVgZtYcNip$uU7-=lpYwKpjh3#eh0iEaKU0$?3}~ z^y1xgkm!%U!a&hu>p3l0Ci9DkZC=Oc&nmEB??omFMTHADtTk8+UajES%qvpNDtPHy X3o}r3>heBDpzz@fQky4Au3!TIn^DAH diff --git a/yaml-tests/src/test/resources/table-functions.metrics.yaml b/yaml-tests/src/test/resources/table-functions.metrics.yaml index 545606b8d6..477815930b 100644 --- a/yaml-tests/src/test/resources/table-functions.metrics.yaml +++ b/yaml-tests/src/test/resources/table-functions.metrics.yaml @@ -1,35 +1,11 @@ table-functions: -- query: EXPLAIN select * from values (1, 2.0, 'foo'), (10, 90.2, 'bar') as A(B, - C, D) - explain: EXPLODE array((@c6 AS B, @c8 AS C, @c10 AS D), (@c14 AS B, @c16 AS C, - @c18 AS D)) - task_count: 62 - task_total_time_ms: 42 - transform_count: 23 - transform_time_ms: 9 - transform_yield_count: 3 - insert_time_ms: 0 - insert_new_count: 3 - insert_reused_count: 0 -- query: EXPLAIN select * from values (1, 2.0, [42, 43, 44]), (11, 3.0, [420, 430, - 440]) - explain: EXPLODE array((@c6 AS _0, @c8 AS _1, array(@c11, @c13, @c15) AS _2), - (@c20 AS _0, @c22 AS _1, array(@c25, @c27, @c29) AS _2)) - task_count: 62 - task_total_time_ms: 2 - transform_count: 23 - transform_time_ms: 0 - transform_yield_count: 3 - insert_time_ms: 0 - insert_new_count: 3 - insert_reused_count: 0 - query: EXPLAIN select * from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, (5, 6.0, 'bar')) as A(B, C, W(X, Y, Z)) explain: EXPLODE array((@c6 AS B, @c8 AS C, (@c11 AS X, promote(@c13 AS DOUBLE) AS Y, @c15 AS Z) AS W), (@c20 AS B, @c22 AS C, (@c25 AS X, @c27 AS Y, @c29 AS Z) AS W)) task_count: 62 - task_total_time_ms: 1 + task_total_time_ms: 3 transform_count: 23 transform_time_ms: 0 transform_yield_count: 3 @@ -46,7 +22,7 @@ table-functions: transform_count: 21 transform_time_ms: 1 transform_yield_count: 3 - insert_time_ms: 1 + insert_time_ms: 0 insert_new_count: 4 insert_reused_count: 0 - query: EXPLAIN select A.B, C, W from values (1, 2.0, (3, 4, 'foo')), (10, 90.2, @@ -55,7 +31,7 @@ table-functions: AS Y, @c21 AS Z) AS W), (@c26 AS B, @c28 AS C, (@c31 AS X, @c33 AS Y, @c35 AS Z) AS W)) | MAP (_.B AS B, _.C AS C, _.W AS W) task_count: 68 - task_total_time_ms: 3 + task_total_time_ms: 2 transform_count: 21 transform_time_ms: 0 transform_yield_count: 3 @@ -81,7 +57,7 @@ table-functions: AS Y, @c29 AS Z) AS W), (@c34 AS B, @c36 AS C, (@c39 AS X, @c41 AS Y, @c43 AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X) task_count: 100 - task_total_time_ms: 5 + task_total_time_ms: 4 transform_count: 33 transform_time_ms: 0 transform_yield_count: 4 @@ -96,10 +72,63 @@ table-functions: AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X) | FILTER _.B LESS_THAN @c68 task_count: 100 - task_total_time_ms: 6 + task_total_time_ms: 7 transform_count: 33 transform_time_ms: 2 transform_yield_count: 4 insert_time_ms: 0 insert_new_count: 6 insert_reused_count: 0 +- query: EXPLAIN select * from range(0, 11, 5) + explain: TFunc range() + task_count: 62 + task_total_time_ms: 1 + transform_count: 23 + transform_time_ms: 0 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 3 + insert_reused_count: 0 +- query: EXPLAIN select * from range(6 - 6, 14 + 6 + 1, 20 - 10) + explain: TFunc range() + task_count: 62 + task_total_time_ms: 0 + transform_count: 23 + transform_time_ms: 0 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 3 + insert_reused_count: 0 +- query: EXPLAIN select ID as X from range(3) as Y + explain: TFunc range() | MAP (_.ID AS X) + task_count: 68 + task_total_time_ms: 1 + transform_count: 21 + transform_time_ms: 0 + transform_yield_count: 3 + insert_time_ms: 0 + insert_new_count: 4 + insert_reused_count: 0 +- query: EXPLAIN select X.ID as A, Y.ID as B from range(3) as X, range(4) as Y + explain: TFunc range() | FLATMAP q0 -> { TFunc range() AS q1 RETURN (q0.ID AS + A, q1.ID AS B) } + task_count: 108 + task_total_time_ms: 19 + transform_count: 33 + transform_time_ms: 6 + transform_yield_count: 6 + insert_time_ms: 1 + insert_new_count: 10 + insert_reused_count: 0 +- query: EXPLAIN select a.id as x, a.col1 as y, b.id as z from t1 as a, range(a.id) + as b + explain: SCAN(<,>) | FLATMAP q0 -> { TFunc range() AS q1 RETURN (q0.ID AS X, q0.COL1 + AS Y, q1.ID AS Z) } + task_count: 131 + task_total_time_ms: 59 + transform_count: 49 + transform_time_ms: 52 + transform_yield_count: 9 + insert_time_ms: 0 + insert_new_count: 8 + insert_reused_count: 0 diff --git a/yaml-tests/src/test/resources/table-functions.yamsql b/yaml-tests/src/test/resources/table-functions.yamsql index ae6e8772d3..6b69d015a6 100644 --- a/yaml-tests/src/test/resources/table-functions.yamsql +++ b/yaml-tests/src/test/resources/table-functions.yamsql @@ -95,15 +95,19 @@ test_block: - result: [{ID: 0}, {ID: 5}, {ID: 10}] - - query: select * from range(0, 11, 5) + - explain: "TFunc range()" - result: [{ID: 0}, {ID: 5}, {ID: 10}] - - query: select * from range(6 - 6, 14 + 6 + 1, 20 - 10) + - explain: "TFunc range()" - result: [{ID: 0}, {ID: 10}, {ID: 20}] - - query: select ID as X from range(3) as Y + - explain: "TFunc range() | MAP (_.ID AS X)" - result: [{X: 0}, {X: 1}, {X: 2}] - - query: select X.ID as A, Y.ID as B from range(3) as X, range(4) as Y + - explain: "TFunc range() | FLATMAP q0 -> { TFunc range() AS q1 RETURN (q0.ID AS A, q1.ID AS B) }" - result: [{A: 0, B: 0}, {A: 0, B: 1}, {A: 0, B: 2}, @@ -118,6 +122,7 @@ test_block: {A: 2, B: 3}] - - query: select a.id as x, a.col1 as y, b.id as z from t1 as a, range(a.id) as b + - explain: "SCAN(<,>) | FLATMAP q0 -> { TFunc range() AS q1 RETURN (q0.ID AS X, q0.COL1 AS Y, q1.ID AS Z) }" - result: [{X: 1, Y: 'a', Z: 0}, {X: 2, Y: 'b', Z: 0}, {X: 2, Y: 'b', Z: 1}, From 446bb3583f35506935e235a79d3951d1f206d619 Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Fri, 28 Mar 2025 17:25:51 +0100 Subject: [PATCH 13/16] remove line. --- yaml-tests/src/test/resources/table-functions.yamsql | 1 - 1 file changed, 1 deletion(-) diff --git a/yaml-tests/src/test/resources/table-functions.yamsql b/yaml-tests/src/test/resources/table-functions.yamsql index 6b69d015a6..3cc8d6cbb5 100644 --- a/yaml-tests/src/test/resources/table-functions.yamsql +++ b/yaml-tests/src/test/resources/table-functions.yamsql @@ -47,7 +47,6 @@ test_block: - explain: "EXPLODE array((@c10 AS B, @c12 AS C, (@c15 AS X, promote(@c17 AS DOUBLE) AS Y, @c19 AS Z) AS W), (@c24 AS B, @c26 AS C, (@c29 AS X, @c31 AS Y, @c33 AS Z) AS W)) | MAP (_.B AS B, _.C AS C, _.W AS W)" - result: [{B: 1, C: 2.0, W: {X: 3, Y: 4.0, Z: 'foo'}}, {B: 10, C: 90.2, W: {X: 5, Y: 6.0, Z: 'bar'}}] -# due to: https://github.com/FoundationDB/fdb-record-layer/issues/3231 - - query: select * from values (1, 2.0, [42, 43, 44]), (11, 3.0, [420, 430, 440]) as A(B, C, W) - result: [{B: 1, C: 2.0, W: [42, 43, 44]}, From 79f9cea525ea613a66b3e614bcac60f2242414cd Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Tue, 1 Apr 2025 15:49:10 +0200 Subject: [PATCH 14/16] address comments. --- .idea/.name | 1 + .../foundationdb/record/RecordCursor.java | 2 +- .../properties/DerivationsProperty.java | 11 +- .../values/FirstOrDefaultStreamingValue.java | 187 ++++++++++++++++++ .../plan/cascades/values/RangeValue.java | 38 +++- .../plan/explain/ExplainPlanVisitor.java | 2 +- .../plans/RecordQueryTableFunctionPlan.java | 2 +- .../src/main/proto/record_query_plan.proto | 5 + .../src/main/antlr/RelationalLexer.g4 | 1 - .../src/test/resources/table-functions.yamsql | 10 +- 10 files changed, 245 insertions(+), 14 deletions(-) create mode 100644 .idea/.name create mode 100644 fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FirstOrDefaultStreamingValue.java diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000000..bd98446bc0 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +fdb-record-layer \ No newline at end of file diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordCursor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordCursor.java index 7d687b1c53..269a4a2e64 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordCursor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordCursor.java @@ -102,7 +102,7 @@ * @param the type of elements of the cursor */ @API(API.Status.UNSTABLE) -public interface RecordCursor extends AutoCloseable { +public interface RecordCursor extends AutoCloseable { /** * The reason that {@link RecordCursorResult#hasNext()} returned false. */ diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java index f755dfe950..6e082b18c8 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java @@ -29,6 +29,7 @@ import com.apple.foundationdb.record.query.plan.cascades.Reference; import com.apple.foundationdb.record.query.plan.cascades.PlanProperty; import com.apple.foundationdb.record.query.plan.cascades.Quantifier; +import com.apple.foundationdb.record.query.plan.cascades.values.FirstOrDefaultStreamingValue; import com.apple.foundationdb.record.query.plan.cascades.values.translation.TranslationMap; import com.apple.foundationdb.record.query.plan.cascades.TreeLike; import com.apple.foundationdb.record.query.plan.cascades.expressions.RelationalExpression; @@ -340,9 +341,9 @@ public Derivations visitInsertPlan(@Nonnull final RecordQueryInsertPlan insertPl @Nonnull @Override public Derivations visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan tableFunctionPlan) { - final var collectionValue = tableFunctionPlan.getValue(); - final var elementType = collectionValue.getResultType(); - final var values = ImmutableList.of(new FirstOrDefaultValue(collectionValue, new ThrowsValue(elementType))); + final var streamingValue = tableFunctionPlan.getValue(); + final var elementType = streamingValue.getResultType(); + final var values = ImmutableList.of(new FirstOrDefaultStreamingValue(streamingValue, new ThrowsValue(elementType))); return new Derivations(values, values); } @@ -797,6 +798,10 @@ public static Derivations evaluateDerivations(@Nonnull RecordQueryPlan recordQue return DERIVATIONS.createVisitor().visit(recordQueryPlan); } + /** + * RCV(T.a, T.b).0 ---> T.a + */ + /** * Cases class to capture the derivations that are being collected by the visitor. */ diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FirstOrDefaultStreamingValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FirstOrDefaultStreamingValue.java new file mode 100644 index 0000000000..abb9c90e5c --- /dev/null +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FirstOrDefaultStreamingValue.java @@ -0,0 +1,187 @@ +/* + * FirstOrDefaultValue.java + * + * This source file is part of the FoundationDB open source project + * + * Copyright 2015-2022 Apple Inc. and the FoundationDB project authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.apple.foundationdb.record.query.plan.cascades.values; + +import com.apple.foundationdb.annotation.API; +import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings; +import com.apple.foundationdb.record.EvaluationContext; +import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanDeserializer; +import com.apple.foundationdb.record.PlanHashable; +import com.apple.foundationdb.record.PlanSerializationContext; +import com.apple.foundationdb.record.RecordCoreException; +import com.apple.foundationdb.record.RecordCursor; +import com.apple.foundationdb.record.logging.LogMessageKeys; +import com.apple.foundationdb.record.planprotos.PFirstOrDefaultStreamingValue; +import com.apple.foundationdb.record.planprotos.PFirstOrDefaultValue; +import com.apple.foundationdb.record.planprotos.PValue; +import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; +import com.apple.foundationdb.record.query.plan.cascades.AliasMap; +import com.apple.foundationdb.record.query.plan.cascades.typing.Type; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; +import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; +import com.google.auto.service.AutoService; +import com.google.common.base.Verify; +import com.google.common.collect.ImmutableList; +import com.google.protobuf.Message; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; + +/** + * A value that returns the first element of the streaming {@link Value} that is passed in. If the streaming value returns + * and empty stream or the first item is null, a default value of the same type is returned. + * + * @see RecordCursor#first() semantics for more information. + */ +@API(API.Status.EXPERIMENTAL) +public class FirstOrDefaultStreamingValue extends AbstractValue { + private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("First-Or-Default-Streaming-Value"); + + @Nonnull + private final StreamingValue childValue; + @Nonnull + private final Value onEmptyResultValue; + @Nonnull + private final Supplier> childrenSupplier; + @Nonnull + private final Type resultType; + + public FirstOrDefaultStreamingValue(@Nonnull final StreamingValue childValue, @Nonnull final Value onEmptyResultValue) { + this.childValue = childValue; + this.onEmptyResultValue = onEmptyResultValue; + this.childrenSupplier = () -> ImmutableList.of(childValue, onEmptyResultValue); + this.resultType = Objects.requireNonNull(childValue.getResultType()); + } + + @Nonnull + @Override + public List computeChildren() { + return childrenSupplier.get(); + } + + @Nonnull + @Override + public Value withChildren(final Iterable newChildren) { + final var newChildrenList = ImmutableList.copyOf(newChildren); + Verify.verify(newChildrenList.size() == 2); + Verify.verify(newChildrenList.get(0) instanceof StreamingValue); + return new FirstOrDefaultStreamingValue((StreamingValue)newChildrenList.get(0), newChildrenList.get(1)); + } + + @Nonnull + @Override + public Type getResultType() { + return resultType; + } + + @Nonnull + public Value getOnEmptyResultValue() { + return onEmptyResultValue; + } + + @Override + @SpotBugsSuppressWarnings({"NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE", "NP_NONNULL_PARAM_VIOLATION"}) + public Object eval(@Nullable final FDBRecordStoreBase store, @Nonnull final EvaluationContext context) { + final var childResult = childValue.evalAsStream(store, context, null, null).first().join(); + if (childResult.isPresent()) { + return childResult.get(); + } else { + return onEmptyResultValue.eval(store, context); + } + } + + @Override + public int hashCodeWithoutChildren() { + return PlanHashable.objectsPlanHash(PlanHashable.CURRENT_FOR_CONTINUATION, BASE_HASH); + } + + @Override + public int planHash(@Nonnull final PlanHashMode mode) { + return PlanHashable.objectsPlanHash(mode, BASE_HASH, childValue, onEmptyResultValue); + } + + @Nonnull + @Override + public ExplainTokensWithPrecedence explain(@Nonnull final Iterable> explainSuppliers) { + return ExplainTokensWithPrecedence.of(new ExplainTokens().addFunctionCall("firstOrDefault", + Value.explainFunctionArguments(explainSuppliers))); + } + + @Override + public int hashCode() { + return semanticHashCode(); + } + + @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") + @SpotBugsSuppressWarnings("EQ_UNUSUAL") + @Override + public boolean equals(final Object other) { + return semanticEquals(other, AliasMap.emptyMap()); + } + + @Nonnull + @Override + public PFirstOrDefaultValue toProto(@Nonnull final PlanSerializationContext serializationContext) { + return PFirstOrDefaultValue.newBuilder() + .setChildValue(childValue.toValueProto(serializationContext)) + .setOnEmptyResultValue(onEmptyResultValue.toValueProto(serializationContext)) + .build(); + } + + @Nonnull + @Override + public PValue toValueProto(@Nonnull PlanSerializationContext serializationContext) { + return PValue.newBuilder().setFirstOrDefaultValue(toProto(serializationContext)).build(); + } + + @Nonnull + public static FirstOrDefaultStreamingValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PFirstOrDefaultStreamingValue firstOrDefaultStreamingValueProto) { + final var value = Value.fromValueProto(serializationContext, Objects.requireNonNull(firstOrDefaultStreamingValueProto.getChildValue())); + if (!(value instanceof StreamingValue)) { + throw new RecordCoreException("invalid value, expecting streaming value").addLogInfo(LogMessageKeys.VALUE, value); + } + return new FirstOrDefaultStreamingValue((StreamingValue)value, + Value.fromValueProto(serializationContext, Objects.requireNonNull(firstOrDefaultStreamingValueProto.getOnEmptyResultValue()))); + } + + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PFirstOrDefaultStreamingValue.class; + } + + @Nonnull + @Override + public FirstOrDefaultStreamingValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PFirstOrDefaultStreamingValue firstOrDefaultValueStreamingProto) { + return FirstOrDefaultStreamingValue.fromProto(serializationContext, firstOrDefaultValueStreamingProto); + } + } +} diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java index 0a375550fc..c196890eff 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java @@ -23,6 +23,7 @@ import com.apple.foundationdb.record.EvaluationContext; import com.apple.foundationdb.record.ExecuteProperties; import com.apple.foundationdb.record.ObjectPlanHash; +import com.apple.foundationdb.record.PlanDeserializer; import com.apple.foundationdb.record.PlanHashable; import com.apple.foundationdb.record.PlanSerializationContext; import com.apple.foundationdb.record.RecordCoreException; @@ -116,13 +117,27 @@ public RecordCursor evalAsStream(@Nonnull final return LiteralValue.ofScalar(rangeValueAsLong); } return v; - })).eval(store, context), continuation); + })).eval(store, context), continuation).skipThenLimit(executeProperties.getSkip(), executeProperties.getReturnedRowLimit()); } @Nonnull @Override public ExplainTokensWithPrecedence explain(@Nonnull final Iterable> explainSuppliers) { - return ExplainTokensWithPrecedence.of(new ExplainTokens().addFunctionCall("range")); // todo improve + final var endExplainTokens = Iterables.get(explainSuppliers, 0).get().getExplainTokens(); + final var beginExplainTokens = + beginInclusive.isEmpty() + ? new ExplainTokens().addKeyword("0") + : Iterables.get(explainSuppliers, 1).get().getExplainTokens(); + final var stepExplainTokens = + step.isEmpty() + ? new ExplainTokens().addKeyword("1") + : Iterables.get(explainSuppliers, 2).get().getExplainTokens(); + + return ExplainTokensWithPrecedence.of(new ExplainTokens().addFunctionCall("range", + new ExplainTokens().addSequence(() -> new ExplainTokens().addCommaAndWhiteSpace(), + beginExplainTokens, + endExplainTokens, + new ExplainTokens().addKeyword("STEP").addWhitespace().addNested(stepExplainTokens)))); } @Nullable @@ -212,6 +227,25 @@ public static RangeValue fromProto(@Nonnull final PlanSerializationContext seria return new RangeValue(endExclusive, beginInclusive, step); } + /** + * Deserializer. + */ + @AutoService(PlanDeserializer.class) + public static class Deserializer implements PlanDeserializer { + @Nonnull + @Override + public Class getProtoMessageClass() { + return PRangeValue.class; + } + + @Nonnull + @Override + public RangeValue fromProto(@Nonnull final PlanSerializationContext serializationContext, + @Nonnull final PRangeValue rangeValueProto) { + return RangeValue.fromProto(serializationContext, rangeValueProto); + } + } + public static class Cursor implements RecordCursor { @Nonnull diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java index c2eeb90ff9..d2ec7c1066 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/explain/ExplainPlanVisitor.java @@ -418,7 +418,7 @@ public ExplainTokens visitInsertPlan(@Nonnull final RecordQueryInsertPlan insert @Nonnull @Override public ExplainTokens visitTableFunctionPlan(@Nonnull final RecordQueryTableFunctionPlan tableFunctionPlan) { - return addKeyword("TFunc").addWhitespace() + return addKeyword("TF").addWhitespace() .addNested(tableFunctionPlan.getValue().explain().getExplainTokens()); } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java index be3807b8a9..09ebc2ccb0 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java @@ -1,5 +1,5 @@ /* - * RecordQueryTableValuedPlan.java + * RecordQueryTableFunctionPlan.java * * This source file is part of the FoundationDB open source project * diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index 2a1331a40a..bf80018b49 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -461,6 +461,11 @@ message PFirstOrDefaultValue { optional PValue on_empty_result_value = 2; } +message PFirstOrDefaultStreamingValue { + optional PValue child_value = 1; + optional PValue on_empty_result_value = 2; +} + message PFromOrderedBytesValue { optional PValue child = 1; optional PDirection direction = 2; diff --git a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 index 54d6109f20..b7892eb08a 100644 --- a/fdb-relational-core/src/main/antlr/RelationalLexer.g4 +++ b/fdb-relational-core/src/main/antlr/RelationalLexer.g4 @@ -166,7 +166,6 @@ PARTITION: 'PARTITION'; PRIMARY: 'PRIMARY'; PROCEDURE: 'PROCEDURE'; PURGE: 'PURGE'; -// RANGE: 'RANGE'; READ: 'READ'; READS: 'READS'; RECURSIVE: 'RECURSIVE'; diff --git a/yaml-tests/src/test/resources/table-functions.yamsql b/yaml-tests/src/test/resources/table-functions.yamsql index 3cc8d6cbb5..eb716a2348 100644 --- a/yaml-tests/src/test/resources/table-functions.yamsql +++ b/yaml-tests/src/test/resources/table-functions.yamsql @@ -94,19 +94,19 @@ test_block: - result: [{ID: 0}, {ID: 5}, {ID: 10}] - - query: select * from range(0, 11, 5) - - explain: "TFunc range()" + - explain: "TF range(promote(@c6 AS LONG), promote(@c8 AS LONG), STEP promote(@c10 AS LONG))" - result: [{ID: 0}, {ID: 5}, {ID: 10}] - - query: select * from range(6 - 6, 14 + 6 + 1, 20 - 10) - - explain: "TFunc range()" + - explain: "TF range(promote(@c6 - @c6 AS LONG), promote(@c10 + @c6 + @c14 AS LONG), STEP promote(@c16 - @c18 AS LONG))" - result: [{ID: 0}, {ID: 10}, {ID: 20}] - - query: select ID as X from range(3) as Y - - explain: "TFunc range() | MAP (_.ID AS X)" + - explain: "TF range(0, promote(@c8 AS LONG), STEP 1) | MAP (_.ID AS X)" - result: [{X: 0}, {X: 1}, {X: 2}] - - query: select X.ID as A, Y.ID as B from range(3) as X, range(4) as Y - - explain: "TFunc range() | FLATMAP q0 -> { TFunc range() AS q1 RETURN (q0.ID AS A, q1.ID AS B) }" + - explain: "TF range(0, promote(@c16 AS LONG), STEP 1) | FLATMAP q0 -> { TF range(0, promote(@c23 AS LONG), STEP 1) AS q1 RETURN (q0.ID AS A, q1.ID AS B) }" - result: [{A: 0, B: 0}, {A: 0, B: 1}, {A: 0, B: 2}, @@ -121,7 +121,7 @@ test_block: {A: 2, B: 3}] - - query: select a.id as x, a.col1 as y, b.id as z from t1 as a, range(a.id) as b - - explain: "SCAN(<,>) | FLATMAP q0 -> { TFunc range() AS q1 RETURN (q0.ID AS X, q0.COL1 AS Y, q1.ID AS Z) }" + - explain: "SCAN(<,>) | FLATMAP q0 -> { TF range(0, q0.ID, STEP 1) AS q1 RETURN (q0.ID AS X, q0.COL1 AS Y, q1.ID AS Z) }" - result: [{X: 1, Y: 'a', Z: 0}, {X: 2, Y: 'b', Z: 0}, {X: 2, Y: 'b', Z: 1}, From 228947a74abf69e7e38e31770bbfec39cd21787d Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Thu, 3 Apr 2025 14:07:28 +0100 Subject: [PATCH 15/16] address comments. --- .idea/.name | 1 - .../foundationdb/record/RecordCursor.java | 2 +- .../expressions/TableFunctionExpression.java | 3 +- .../properties/DerivationsProperty.java | 4 - .../rules/ImplementTableFunctionRule.java | 7 +- .../values/FirstOrDefaultStreamingValue.java | 2 +- .../plan/cascades/values/RangeValue.java | 114 +++++++----------- .../plan/cascades/values/StreamingValue.java | 9 ++ .../plan/plans/RecordQueryExplodePlan.java | 5 - .../plans/RecordQueryTableFunctionPlan.java | 3 +- .../src/main/proto/record_cursor.proto | 2 +- .../src/main/proto/record_query_plan.proto | 1 + .../src/test/java/YamlIntegrationTests.java | 1 + .../resources/table-functions.metrics.binpb | Bin 13845 -> 14400 bytes .../resources/table-functions.metrics.yaml | 36 +++--- .../src/test/resources/table-functions.yamsql | 6 +- 16 files changed, 85 insertions(+), 111 deletions(-) delete mode 100644 .idea/.name diff --git a/.idea/.name b/.idea/.name deleted file mode 100644 index bd98446bc0..0000000000 --- a/.idea/.name +++ /dev/null @@ -1 +0,0 @@ -fdb-record-layer \ No newline at end of file diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordCursor.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordCursor.java index 269a4a2e64..7d687b1c53 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordCursor.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordCursor.java @@ -102,7 +102,7 @@ * @param the type of elements of the cursor */ @API(API.Status.UNSTABLE) -public interface RecordCursor extends AutoCloseable { +public interface RecordCursor extends AutoCloseable { /** * The reason that {@link RecordCursorResult#hasNext()} returned false. */ diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TableFunctionExpression.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TableFunctionExpression.java index cee2333511..7cc842e14d 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TableFunctionExpression.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/expressions/TableFunctionExpression.java @@ -104,8 +104,7 @@ public boolean equalsWithoutChildren(@Nonnull RelationalExpression otherExpressi final var otherTableFunctionExpression = (TableFunctionExpression)otherExpression; - return value.semanticEquals(otherTableFunctionExpression.getValue(), equivalencesMap) && - semanticEqualsForResults(otherExpression, equivalencesMap); + return value.semanticEquals(otherTableFunctionExpression.getValue(), equivalencesMap); } @Override diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java index 6e082b18c8..92b7493ea2 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/properties/DerivationsProperty.java @@ -798,10 +798,6 @@ public static Derivations evaluateDerivations(@Nonnull RecordQueryPlan recordQue return DERIVATIONS.createVisitor().visit(recordQueryPlan); } - /** - * RCV(T.a, T.b).0 ---> T.a - */ - /** * Cases class to capture the derivations that are being collected by the visitor. */ diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTableFunctionRule.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTableFunctionRule.java index 7a8360dc07..d9d88930b4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTableFunctionRule.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/rules/ImplementTableFunctionRule.java @@ -32,10 +32,9 @@ import static com.apple.foundationdb.record.query.plan.cascades.matching.structure.RelationalExpressionMatchers.tableFunctionExpression; /** - * A rule that implements an table function expression into a {@link RecordQueryTableFunctionPlan}. + * A rule that implements a table function expression into a {@link RecordQueryTableFunctionPlan}. */ @API(API.Status.EXPERIMENTAL) -@SuppressWarnings("PMD.TooManyStaticImports") public class ImplementTableFunctionRule extends CascadesRule { private static final BindingMatcher root = tableFunctionExpression(); @@ -46,7 +45,7 @@ public ImplementTableFunctionRule() { @Override public void onMatch(@Nonnull final CascadesRuleCall call) { - final var explodeExpression = call.get(root); - call.yieldExpression(new RecordQueryTableFunctionPlan(explodeExpression.getValue())); + final var tableFunctionExpression = call.get(root); + call.yieldExpression(new RecordQueryTableFunctionPlan(tableFunctionExpression.getValue())); } } diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FirstOrDefaultStreamingValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FirstOrDefaultStreamingValue.java index abb9c90e5c..e1ece731e4 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FirstOrDefaultStreamingValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/FirstOrDefaultStreamingValue.java @@ -1,5 +1,5 @@ /* - * FirstOrDefaultValue.java + * FirstOrDefaultStreamingValue.java * * This source file is part of the FoundationDB open source project * diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java index c196890eff..c7e4718759 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/RangeValue.java @@ -73,7 +73,6 @@ * * For more information, see relational SQL {@code range} table-valued function. */ -@SuppressWarnings("OptionalUsedAsFieldOrParameterType") public class RangeValue extends AbstractValue implements StreamingValue, CreatesDynamicTypesValue { private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Range-Value"); @@ -81,16 +80,16 @@ public class RangeValue extends AbstractValue implements StreamingValue, Creates private final Value endExclusive; @Nonnull - private final Optional beginInclusive; + private final Value beginInclusive; @Nonnull - private final Optional step; + private final Value step; @Nonnull private final Value currentRangeValue; - public RangeValue(@Nonnull final Value endExclusive, @Nonnull final Optional beginInclusive, - @Nonnull final Optional step) { + public RangeValue(@Nonnull final Value endExclusive, @Nonnull final Value beginInclusive, + @Nonnull final Value step) { this.endExclusive = endExclusive; this.beginInclusive = beginInclusive; this.step = step; @@ -110,9 +109,9 @@ public RecordCursor evalAsStream(@Nonnull final @Nullable final byte[] continuation, @Nonnull final ExecuteProperties executeProperties) { final long endExclusiveValue = (Long)Verify.verifyNotNull(endExclusive.eval(store, context)); - final var beginInclusiveMaybe = beginInclusive.map(b -> (Long)Verify.verifyNotNull(b.eval(store, context))); - final var stepMaybe = step.map(s -> (Long)Verify.verifyNotNull(s.eval(store, context))); - return new Cursor(store.getExecutor(), endExclusiveValue, beginInclusiveMaybe, stepMaybe, rangeValueAsLong -> Objects.requireNonNull(currentRangeValue.replace(v -> { + final var beginInclusiveValue = (Long)Verify.verifyNotNull(this.beginInclusive.eval(store, context)); + final var stepValue = (Long)Verify.verifyNotNull(step.eval(store, context)); + return new Cursor(store.getExecutor(), endExclusiveValue, beginInclusiveValue, stepValue, rangeValueAsLong -> Objects.requireNonNull(currentRangeValue.replace(v -> { if (v instanceof LiteralValue) { return LiteralValue.ofScalar(rangeValueAsLong); } @@ -124,14 +123,8 @@ public RecordCursor evalAsStream(@Nonnull final @Override public ExplainTokensWithPrecedence explain(@Nonnull final Iterable> explainSuppliers) { final var endExplainTokens = Iterables.get(explainSuppliers, 0).get().getExplainTokens(); - final var beginExplainTokens = - beginInclusive.isEmpty() - ? new ExplainTokens().addKeyword("0") - : Iterables.get(explainSuppliers, 1).get().getExplainTokens(); - final var stepExplainTokens = - step.isEmpty() - ? new ExplainTokens().addKeyword("1") - : Iterables.get(explainSuppliers, 2).get().getExplainTokens(); + final var beginExplainTokens = Iterables.get(explainSuppliers, 1).get().getExplainTokens(); + final var stepExplainTokens = Iterables.get(explainSuppliers, 2).get().getExplainTokens(); return ExplainTokensWithPrecedence.of(new ExplainTokens().addFunctionCall("range", new ExplainTokens().addSequence(() -> new ExplainTokens().addCommaAndWhiteSpace(), @@ -167,8 +160,8 @@ public int planHash(@Nonnull final PlanHashMode hashMode) { public PRangeValue toProto(@Nonnull final PlanSerializationContext serializationContext) { final PRangeValue.Builder builder = PRangeValue.newBuilder(); builder.setEndExclusiveChild(endExclusive.toValueProto(serializationContext)); - beginInclusive.ifPresent(b -> builder.setBeginInclusiveChild(b.toValueProto(serializationContext))); - step.ifPresent(s -> builder.setStepChild(s.toValueProto(serializationContext))); + builder.setBeginInclusiveChild(beginInclusive.toValueProto(serializationContext)); + builder.setStepChild(step.toValueProto(serializationContext)); return builder.build(); } @@ -177,8 +170,8 @@ public PRangeValue toProto(@Nonnull final PlanSerializationContext serialization protected Iterable computeChildren() { final ImmutableList.Builder childrenBuilder = ImmutableList.builder(); childrenBuilder.add(endExclusive); - beginInclusive.ifPresent(childrenBuilder::add); - step.ifPresent(childrenBuilder::add); + childrenBuilder.add(beginInclusive); + childrenBuilder.add(step); return childrenBuilder.build(); } @@ -187,30 +180,15 @@ protected Iterable computeChildren() { @SuppressWarnings("PMD.CompareObjectsWithEquals") // intentional. public Value withChildren(final Iterable newChildren) { final var newChildrenSize = Iterables.size(newChildren); - Verify.verify(newChildrenSize <= 3 && newChildrenSize > 0); + Verify.verify(newChildrenSize == 3); final var newEndExclusive = Iterables.get(newChildren, 0); + final var newBeginInclusive = Iterables.get(newChildren, 1); + final var newStep = Iterables.get(newChildren, 2); - if (newChildrenSize == 1 && newEndExclusive == this.endExclusive) { + if (newEndExclusive == endExclusive && newBeginInclusive == beginInclusive && newStep == step) { return this; } - Optional newBeginInclusive = Optional.empty(); - if (newChildrenSize > 1) { - Verify.verify(beginInclusive.isPresent()); - newBeginInclusive = Optional.of(Iterables.get(newChildren, 1)); - if (newChildrenSize == 2 && newEndExclusive == this.endExclusive && newBeginInclusive.get() == beginInclusive.get()) { - return this; - } - } - - Optional newStep = Optional.empty(); - if (newChildrenSize > 2) { - Verify.verify(step.isPresent()); - newStep = Optional.of(Iterables.get(newChildren, 2)); - if (newEndExclusive == this.endExclusive && newBeginInclusive.get() == beginInclusive.get() && newStep.get() == step.get()) { - return this; - } - } return new RangeValue(newEndExclusive, newBeginInclusive, newStep); } @@ -218,12 +196,8 @@ public Value withChildren(final Iterable newChildren) { public static RangeValue fromProto(@Nonnull final PlanSerializationContext serializationContext, @Nonnull final PRangeValue rangeValueProto) { final var endExclusive = Value.fromValueProto(serializationContext, rangeValueProto.getEndExclusiveChild()); - final Optional beginInclusive = rangeValueProto.hasBeginInclusiveChild() - ? Optional.of(Value.fromValueProto(serializationContext, rangeValueProto.getBeginInclusiveChild())) - : Optional.empty(); - final Optional step = rangeValueProto.hasStepChild() - ? Optional.of(Value.fromValueProto(serializationContext, rangeValueProto.getStepChild())) - : Optional.empty(); + final var beginInclusive = Value.fromValueProto(serializationContext, rangeValueProto.getBeginInclusiveChild()); + final var step = Value.fromValueProto(serializationContext, rangeValueProto.getStepChild()); return new RangeValue(endExclusive, beginInclusive, step); } @@ -253,8 +227,7 @@ public static class Cursor implements RecordCursor { private final long endExclusive; - @Nonnull - private final Optional step; + private final long step; private long nextPosition; // position of the next value to return @@ -263,13 +236,13 @@ public static class Cursor implements RecordCursor { @Nonnull private final Function rangeValueCreator; - Cursor(@Nonnull final Executor executor, final long endExclusive, @Nonnull final Optional beginInclusive, - @Nonnull final Optional step, @Nonnull final Function rangeValueCreator, @Nullable final byte[] continuation) { + Cursor(@Nonnull final Executor executor, final long endExclusive, final long beginInclusive, + final long step, @Nonnull final Function rangeValueCreator, @Nullable final byte[] continuation) { this(executor, endExclusive, step, rangeValueCreator, - continuation == null ? beginInclusive.orElse(0L) : Continuation.from(continuation, endExclusive, step).getNextPosition()); + continuation == null ? beginInclusive : Continuation.from(continuation, endExclusive, step).getNextPosition()); } - private Cursor(@Nonnull final Executor executor, final long endExclusive, @Nonnull final Optional step, + private Cursor(@Nonnull final Executor executor, final long endExclusive, final long step, @Nonnull final Function rangeValueCreator, final long nextPosition) { checkValidRange(nextPosition, endExclusive, step); this.executor = executor; @@ -290,9 +263,9 @@ public CompletableFuture> onNext() { public RecordCursorResult getNext() { RecordCursorResult nextResult; if (nextPosition < endExclusive) { - final var continuation = new Continuation(endExclusive, nextPosition + step.orElse(1L), step); + final var continuation = new Continuation(endExclusive, nextPosition + step, step); nextResult = RecordCursorResult.withNextValue(QueryResult.ofComputed(rangeValueCreator.apply(nextPosition)), continuation); - nextPosition += step.orElse(1L); + nextPosition += step; } else { nextResult = RecordCursorResult.exhausted(); } @@ -321,14 +294,14 @@ public Executor getExecutor() { return executor; } - private static void checkValidRange(long position, long endExclusive, @Nonnull final Optional step) { - if (position < 0) { + private static void checkValidRange(final long position, final long endExclusive, final long step) { + if (position < 0L) { throw new RecordCoreException("only non-negative position is allowed in range"); } - if (endExclusive < 0) { + if (endExclusive < 0L) { throw new RecordCoreException("only non-negative exclusive end is allowed in range"); } - if (step.isPresent() && step.get() <= 0) { + if (step <= 0L) { throw new RecordCoreException("only positive step is allowed in range"); } } @@ -336,10 +309,9 @@ private static void checkValidRange(long position, long endExclusive, @Nonnull f private static class Continuation implements RecordCursorContinuation { private final long endExclusive; private final long nextPosition; - @Nonnull - private final Optional step; + private final long step; - public Continuation(long endExclusive, long nextPosition, @Nonnull final Optional step) { + public Continuation(final long endExclusive, final long nextPosition, final long step) { this.nextPosition = nextPosition; this.endExclusive = endExclusive; this.step = step; @@ -350,7 +322,7 @@ public boolean isEnd() { // If a next value is returned as part of a cursor result, the continuation must not be an end continuation // (i.e., isEnd() must be false), per the contract of RecordCursorResult. This is the case even if the // cursor knows for certain that there is no more after that result, as in the ListCursor. - return nextPosition >= endExclusive + step.orElse(1L); + return nextPosition >= endExclusive + step; } public long getNextPosition() { @@ -374,13 +346,15 @@ public byte[] toBytes() { } @Nonnull - public static Continuation from(@Nonnull final RecordCursorProto.RangeCursorContinuation message, long endExclusive, @Nonnull final Optional step) { + public static Continuation from(@Nonnull final RecordCursorProto.RangeCursorContinuation message, + final long endExclusive, final long step) { final var nextPosition = message.getNextPosition(); return new Continuation(endExclusive, nextPosition, step); } @Nonnull - public static Continuation from(@Nonnull final byte[] unparsedContinuationBytes, long endExclusive, @Nonnull final Optional step) { + public static Continuation from(@Nonnull final byte[] unparsedContinuationBytes, final long endExclusive, + final long step) { try { final var parsed = RecordCursorProto.RangeCursorContinuation.parseFrom(unparsedContinuationBytes); return from(parsed, endExclusive, step); @@ -402,23 +376,23 @@ public static class RangeFn extends BuiltInTableFunction { private static StreamingValue encapsulateInternal(@Nonnull final List arguments) { Verify.verify(!arguments.isEmpty()); final Value endExclusive; - final Optional beginInclusiveMaybe; - final Optional stepMaybe; + final Value beginInclusiveMaybe; + final Value stepMaybe; if (arguments.size() == 1) { endExclusive = PromoteValue.inject((Value)arguments.get(0), Type.primitiveType(Type.TypeCode.LONG)); checkValidBoundaryType(endExclusive); - beginInclusiveMaybe = Optional.empty(); - stepMaybe = Optional.empty(); + beginInclusiveMaybe = LiteralValue.ofScalar(0L); + stepMaybe = LiteralValue.ofScalar(1L); } else { checkValidBoundaryType((Value)arguments.get(0)); - beginInclusiveMaybe = Optional.of(PromoteValue.inject((Value)arguments.get(0), Type.primitiveType(Type.TypeCode.LONG))); + beginInclusiveMaybe = PromoteValue.inject((Value)arguments.get(0), Type.primitiveType(Type.TypeCode.LONG)); checkValidBoundaryType((Value)arguments.get(1)); endExclusive = PromoteValue.inject((Value)arguments.get(1), Type.primitiveType(Type.TypeCode.LONG)); if (arguments.size() > 2) { checkValidBoundaryType((Value)arguments.get(2)); - stepMaybe = Optional.of(PromoteValue.inject((Value)arguments.get(2), Type.primitiveType(Type.TypeCode.LONG))); + stepMaybe = PromoteValue.inject((Value)arguments.get(2), Type.primitiveType(Type.TypeCode.LONG)); } else { - stepMaybe = Optional.empty(); + stepMaybe = LiteralValue.ofScalar(1L); } } return new RangeValue(endExclusive, beginInclusiveMaybe, stepMaybe); diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java index 163a78c1ff..c70ad58e20 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/values/StreamingValue.java @@ -35,6 +35,15 @@ */ public interface StreamingValue extends Value { + /** + * Returns a {@link RecordCursor} over the result stream returned upon evaluation. + * @param store The record store used to fetch and evaluate the results. + * @param context The evaluation context, containing a set of contextual parameters for evaluating the results. + * @param continuation The continuation bytes used to resume the evaluation of the result stream. + * @param executeProperties The execution properties. + * @param The type of the returned results when fetched from a record store. + * @return A cursor over the result stream returned upon evaluation. + */ @Nonnull RecordCursor evalAsStream(@Nonnull FDBRecordStoreBase store, @Nonnull EvaluationContext context, diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java index 0c3d921178..2935edd3b9 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryExplodePlan.java @@ -33,7 +33,6 @@ import com.apple.foundationdb.record.provider.common.StoreTimer; import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.query.plan.AvailableFields; -import com.apple.foundationdb.record.query.plan.cascades.values.StreamingValue; import com.apple.foundationdb.record.query.plan.explain.ExplainPlanVisitor; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; import com.apple.foundationdb.record.query.plan.cascades.CorrelationIdentifier; @@ -87,10 +86,6 @@ public RecordCursor executePlan(@Nonnull final @Nonnull final EvaluationContext context, @Nullable final byte[] continuation, @Nonnull final ExecuteProperties executeProperties) { - if (collectionValue instanceof StreamingValue) { - final var streamingValue = (StreamingValue)collectionValue; - return streamingValue.evalAsStream(store, context, continuation, executeProperties); - } final var result = collectionValue.eval(store, context); return RecordCursor.fromList(result == null ? ImmutableList.of() : (List)result, continuation) .map(QueryResult::ofComputed) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java index 09ebc2ccb0..45e8d323d7 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/plans/RecordQueryTableFunctionPlan.java @@ -181,8 +181,7 @@ public boolean equalsWithoutChildren(@Nonnull final RelationalExpression otherEx } final var otherTableFunctionPlan = (RecordQueryTableFunctionPlan)otherExpression; - return value.semanticEquals(otherTableFunctionPlan.value, equivalencesMap) && - semanticEqualsForResults(otherExpression, equivalencesMap); + return value.semanticEquals(otherTableFunctionPlan.value, equivalencesMap); } @SuppressWarnings("EqualsWhichDoesntCheckParameterClass") diff --git a/fdb-record-layer-core/src/main/proto/record_cursor.proto b/fdb-record-layer-core/src/main/proto/record_cursor.proto index 4cb306a829..5c93228902 100644 --- a/fdb-record-layer-core/src/main/proto/record_cursor.proto +++ b/fdb-record-layer-core/src/main/proto/record_cursor.proto @@ -134,5 +134,5 @@ message RecursiveCursorContinuation { } message RangeCursorContinuation { - optional int64 nextPosition = 2; + optional int64 nextPosition = 1; } \ No newline at end of file diff --git a/fdb-record-layer-core/src/main/proto/record_query_plan.proto b/fdb-record-layer-core/src/main/proto/record_query_plan.proto index bf80018b49..33bcf47d8c 100644 --- a/fdb-record-layer-core/src/main/proto/record_query_plan.proto +++ b/fdb-record-layer-core/src/main/proto/record_query_plan.proto @@ -252,6 +252,7 @@ message PValue { PQuantifiedRecordValue quantified_record_value = 46; PMacroFunctionValue macro_function_value = 47; PRangeValue range_value = 48; + PFirstOrDefaultStreamingValue first_or_default_streaming_value = 49; } } diff --git a/yaml-tests/src/test/java/YamlIntegrationTests.java b/yaml-tests/src/test/java/YamlIntegrationTests.java index c2d1e54e78..4ddee2bb44 100644 --- a/yaml-tests/src/test/java/YamlIntegrationTests.java +++ b/yaml-tests/src/test/java/YamlIntegrationTests.java @@ -260,6 +260,7 @@ public void uuidTest(YamlTest.Runner runner) throws Exception { } @TestTemplate + @MaintainYamlTestConfig(YamlTestConfigFilters.CORRECT_EXPLAIN_AND_METRICS) public void tableFunctionsTest(YamlTest.Runner runner) throws Exception { runner.runYamsql("table-functions.yamsql"); } diff --git a/yaml-tests/src/test/resources/table-functions.metrics.binpb b/yaml-tests/src/test/resources/table-functions.metrics.binpb index d2cd781e7cf311d39c479770041073b17ea09513..9396e2076059b2b5aaaed8248e5c5366df18f983 100644 GIT binary patch delta 1161 zcmbQ5bD&^CyTI+{b&L|?3UiwzG?)#(9OK$J-G*8G!=?{h5~2#vzJve;51kjYU~ym& zI?6ctBHMz=w)~Qte{wo7P5#5&w|Ty}2IJ(J91{d4%v#6@)N$a8Fi^*?-Q1fkIel4$ z-d>yp5?ypr04Tcd3%do&WPTB`&FlF5Sp`n6Sj8lvsPO26hX#wmt#_Q8c}0p@1s_j+ z&kPiOw`DscQ26Lnzs(aRSFkZXWtnWPU|qk0g-eXXPGJA@G@z58-r)qAI`I#)1vAif z0U>S*MTvRosTu`E`MLQesTvN+W(tnM3O@dR?wUFZ2vG|p(clo*03<0x1Gt=~)NbaK z%=Dtff(!+Bg=#JZg|z(1{)|T__p9-^A`GLzipd+8S|&Sa3ozHNWt}Xk!S*cKiJ z203`+(=capfr<#zeKxQ!CdhDXPFA&GR3xlxBU3XlLWCw8s83ZcQOd4zk(^uGAZ~eCa`geFl`rx+Qd#}82T zIz7qBEgFJMYq^2GSUR~^-*R)QrU#=Es#dHPLA`jZl<8>=oIpX!3;-0QCA~h5!Hn delta 617 zcmX?5Fg0gFyTFsJR~RM46^`#v(_l9E-y*nix(&1VqG^p>5~2!MemVgKKedZlusARX z9c7$+k!`_bTYkySKRF$kCjVjX+dN-fgK_dqjtK(aUan#U>iBeB7^q{xX3@=-oW86= zH=a!giN4<_1QeY!fy;trGQWt}=5>7jtOAG6&18~LROnf1p}}Hsd^Oi*UXfx}!R0;2 znSr9qHm_s^3O~Fiw|S!E3O1%Y%#*DZtaauybBS@-3AEnw0Xn(m0VmMZAC0UQ%s|)i zhPajHB`Xvq=B1}!DK%bp2-`TnkV-w2{OH7p1erO7iP}u3uZub zroRN4^KLN<@#Zk@o*ba+VE2m+=o}Y;(^t}go}B-h2WZT~^X%Zjl1FilLXCp2V}OE2 zyq>3vf@82kgr?L!HmCzFHlI|pV1&9*U~-S1?PLdzNTyvplYKR#z3y^zNpR!{9NXQ+ z2@IBP+x|0Yuo--v;Ap`G^m8b>C2l^BAs~wi4HR_k6si@FAGn$5{3$!`~FF2a2kA{s9?bXv^swB f1OquPM2G`Tmz?~-K%Q~!Cd0)naPvIOelP+6b}q^a diff --git a/yaml-tests/src/test/resources/table-functions.metrics.yaml b/yaml-tests/src/test/resources/table-functions.metrics.yaml index 477815930b..89ee335c7f 100644 --- a/yaml-tests/src/test/resources/table-functions.metrics.yaml +++ b/yaml-tests/src/test/resources/table-functions.metrics.yaml @@ -5,7 +5,7 @@ table-functions: AS Y, @c15 AS Z) AS W), (@c20 AS B, @c22 AS C, (@c25 AS X, @c27 AS Y, @c29 AS Z) AS W)) task_count: 62 - task_total_time_ms: 3 + task_total_time_ms: 2 transform_count: 23 transform_time_ms: 0 transform_yield_count: 3 @@ -18,7 +18,7 @@ table-functions: AS Y, @c19 AS Z) AS W), (@c24 AS B, @c26 AS C, (@c29 AS X, @c31 AS Y, @c33 AS Z) AS W)) | MAP (_.B AS B, _.C AS C, _.W AS W) task_count: 68 - task_total_time_ms: 20 + task_total_time_ms: 22 transform_count: 21 transform_time_ms: 1 transform_yield_count: 3 @@ -59,7 +59,7 @@ table-functions: task_count: 100 task_total_time_ms: 4 transform_count: 33 - transform_time_ms: 0 + transform_time_ms: 1 transform_yield_count: 4 insert_time_ms: 0 insert_new_count: 5 @@ -72,15 +72,16 @@ table-functions: AS Z) AS W)) | MAP (_.B AS B, _.C AS Q, _.W.X AS X) | FILTER _.B LESS_THAN @c68 task_count: 100 - task_total_time_ms: 7 + task_total_time_ms: 8 transform_count: 33 - transform_time_ms: 2 + transform_time_ms: 3 transform_yield_count: 4 - insert_time_ms: 0 + insert_time_ms: 1 insert_new_count: 6 insert_reused_count: 0 - query: EXPLAIN select * from range(0, 11, 5) - explain: TFunc range() + explain: TF range(promote(@c6 AS LONG), promote(@c8 AS LONG), STEP promote(@c10 + AS LONG)) task_count: 62 task_total_time_ms: 1 transform_count: 23 @@ -90,7 +91,8 @@ table-functions: insert_new_count: 3 insert_reused_count: 0 - query: EXPLAIN select * from range(6 - 6, 14 + 6 + 1, 20 - 10) - explain: TFunc range() + explain: TF range(promote(@c6 - @c6 AS LONG), promote(@c10 + @c6 + @c14 AS LONG), + STEP promote(@c16 - @c18 AS LONG)) task_count: 62 task_total_time_ms: 0 transform_count: 23 @@ -100,7 +102,7 @@ table-functions: insert_new_count: 3 insert_reused_count: 0 - query: EXPLAIN select ID as X from range(3) as Y - explain: TFunc range() | MAP (_.ID AS X) + explain: TF range(0l, promote(@c8 AS LONG), STEP 1l) | MAP (_.ID AS X) task_count: 68 task_total_time_ms: 1 transform_count: 21 @@ -110,24 +112,24 @@ table-functions: insert_new_count: 4 insert_reused_count: 0 - query: EXPLAIN select X.ID as A, Y.ID as B from range(3) as X, range(4) as Y - explain: TFunc range() | FLATMAP q0 -> { TFunc range() AS q1 RETURN (q0.ID AS - A, q1.ID AS B) } + explain: TF range(0l, promote(@c16 AS LONG), STEP 1l) | FLATMAP q0 -> { TF range(0l, + promote(@c23 AS LONG), STEP 1l) AS q1 RETURN (q0.ID AS A, q1.ID AS B) } task_count: 108 - task_total_time_ms: 19 + task_total_time_ms: 25 transform_count: 33 - transform_time_ms: 6 + transform_time_ms: 5 transform_yield_count: 6 insert_time_ms: 1 insert_new_count: 10 insert_reused_count: 0 - query: EXPLAIN select a.id as x, a.col1 as y, b.id as z from t1 as a, range(a.id) as b - explain: SCAN(<,>) | FLATMAP q0 -> { TFunc range() AS q1 RETURN (q0.ID AS X, q0.COL1 - AS Y, q1.ID AS Z) } + explain: SCAN(<,>) | FLATMAP q0 -> { TF range(0l, q0.ID, STEP 1l) AS q1 RETURN + (q0.ID AS X, q0.COL1 AS Y, q1.ID AS Z) } task_count: 131 - task_total_time_ms: 59 + task_total_time_ms: 55 transform_count: 49 - transform_time_ms: 52 + transform_time_ms: 46 transform_yield_count: 9 insert_time_ms: 0 insert_new_count: 8 diff --git a/yaml-tests/src/test/resources/table-functions.yamsql b/yaml-tests/src/test/resources/table-functions.yamsql index eb716a2348..e2d876c973 100644 --- a/yaml-tests/src/test/resources/table-functions.yamsql +++ b/yaml-tests/src/test/resources/table-functions.yamsql @@ -102,11 +102,11 @@ test_block: - result: [{ID: 0}, {ID: 10}, {ID: 20}] - - query: select ID as X from range(3) as Y - - explain: "TF range(0, promote(@c8 AS LONG), STEP 1) | MAP (_.ID AS X)" + - explain: "TF range(0l, promote(@c8 AS LONG), STEP 1l) | MAP (_.ID AS X)" - result: [{X: 0}, {X: 1}, {X: 2}] - - query: select X.ID as A, Y.ID as B from range(3) as X, range(4) as Y - - explain: "TF range(0, promote(@c16 AS LONG), STEP 1) | FLATMAP q0 -> { TF range(0, promote(@c23 AS LONG), STEP 1) AS q1 RETURN (q0.ID AS A, q1.ID AS B) }" + - explain: "TF range(0l, promote(@c16 AS LONG), STEP 1l) | FLATMAP q0 -> { TF range(0l, promote(@c23 AS LONG), STEP 1l) AS q1 RETURN (q0.ID AS A, q1.ID AS B) }" - result: [{A: 0, B: 0}, {A: 0, B: 1}, {A: 0, B: 2}, @@ -121,7 +121,7 @@ test_block: {A: 2, B: 3}] - - query: select a.id as x, a.col1 as y, b.id as z from t1 as a, range(a.id) as b - - explain: "SCAN(<,>) | FLATMAP q0 -> { TF range(0, q0.ID, STEP 1) AS q1 RETURN (q0.ID AS X, q0.COL1 AS Y, q1.ID AS Z) }" + - explain: "SCAN(<,>) | FLATMAP q0 -> { TF range(0l, q0.ID, STEP 1l) AS q1 RETURN (q0.ID AS X, q0.COL1 AS Y, q1.ID AS Z) }" - result: [{X: 1, Y: 'a', Z: 0}, {X: 2, Y: 'b', Z: 0}, {X: 2, Y: 'b', Z: 1}, From ca8e19c99c85944d1b29b613f236b95c0945ed7a Mon Sep 17 00:00:00 2001 From: Youssef Hatem Date: Thu, 3 Apr 2025 14:52:52 +0100 Subject: [PATCH 16/16] address further comments. --- .../cascades/predicates/CompatibleTypeEvolutionPredicate.java | 3 ++- yaml-tests/src/test/java/YamlIntegrationTests.java | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/CompatibleTypeEvolutionPredicate.java b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/CompatibleTypeEvolutionPredicate.java index 0399e57daf..8896be18d1 100644 --- a/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/CompatibleTypeEvolutionPredicate.java +++ b/fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/predicates/CompatibleTypeEvolutionPredicate.java @@ -37,6 +37,7 @@ import com.apple.foundationdb.record.provider.foundationdb.FDBRecordStoreBase; import com.apple.foundationdb.record.query.plan.cascades.AliasMap; import com.apple.foundationdb.record.query.plan.cascades.BooleanWithConstraint; +import com.apple.foundationdb.record.query.plan.cascades.values.FirstOrDefaultStreamingValue; import com.apple.foundationdb.record.query.plan.explain.ExplainTokens; import com.apple.foundationdb.record.query.plan.explain.ExplainTokensWithPrecedence; import com.apple.foundationdb.record.query.plan.cascades.ValueEquivalence; @@ -290,7 +291,7 @@ private static List computeFieldAccessForDerivation( return resultTrieBuilders.build(); } - if (derivationValue instanceof FirstOrDefaultValue) { + if (derivationValue instanceof FirstOrDefaultValue || derivationValue instanceof FirstOrDefaultStreamingValue) { Verify.verify(nestedResults.size() == 2); return nestedResults.get(0); } diff --git a/yaml-tests/src/test/java/YamlIntegrationTests.java b/yaml-tests/src/test/java/YamlIntegrationTests.java index 4ddee2bb44..c2d1e54e78 100644 --- a/yaml-tests/src/test/java/YamlIntegrationTests.java +++ b/yaml-tests/src/test/java/YamlIntegrationTests.java @@ -260,7 +260,6 @@ public void uuidTest(YamlTest.Runner runner) throws Exception { } @TestTemplate - @MaintainYamlTestConfig(YamlTestConfigFilters.CORRECT_EXPLAIN_AND_METRICS) public void tableFunctionsTest(YamlTest.Runner runner) throws Exception { runner.runYamsql("table-functions.yamsql"); }