Support UDF in plan generator

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaData.java

@@ -28,6 +28,7 @@
 import com.apple.foundationdb.record.metadata.MetaDataException;
 import com.apple.foundationdb.record.metadata.RecordType;
 import com.apple.foundationdb.record.metadata.SyntheticRecordType;
+import com.apple.foundationdb.record.metadata.ScalarValuedFunction;
 import com.apple.foundationdb.record.metadata.UnnestedRecordType;
 import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
 import com.apple.foundationdb.record.metadata.expressions.LiteralKeyExpression;
@@ -48,6 +49,7 @@
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.TreeMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
@@ -83,6 +85,8 @@ public class RecordMetaData implements RecordMetaDataProvider {
     @Nonnull
     private final Map<Object, SyntheticRecordType<?>> recordTypeKeyToSyntheticTypeMap;
     @Nonnull
+    private final Set<ScalarValuedFunction> scalarValuedFunctions;
+    @Nonnull
     private final Map<String, Index> indexes;
     @Nonnull
     private final Map<String, Index> universalIndexes;
@@ -112,6 +116,7 @@ protected RecordMetaData(@Nonnull RecordMetaData orig) {
                 Collections.unmodifiableMap(orig.indexes),
                 Collections.unmodifiableMap(orig.universalIndexes),
                 Collections.unmodifiableList(orig.formerIndexes),
+                Collections.unmodifiableSet(orig.scalarValuedFunctions),
                 orig.splitLongRecords,
                 orig.storeRecordVersions,
                 orig.version,
@@ -131,6 +136,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor,
                              @Nonnull Map<String, Index> indexes,
                              @Nonnull Map<String, Index> universalIndexes,
                              @Nonnull List<FormerIndex> formerIndexes,
+                             @Nonnull Set<ScalarValuedFunction> scalarValuedFunctions,
                              boolean splitLongRecords,
                              boolean storeRecordVersions,
                              int version,
@@ -147,6 +153,7 @@ protected RecordMetaData(@Nonnull Descriptors.FileDescriptor recordsDescriptor,
         this.indexes = indexes;
         this.universalIndexes = universalIndexes;
         this.formerIndexes = formerIndexes;
+        this.scalarValuedFunctions = scalarValuedFunctions;
         this.splitLongRecords = splitLongRecords;
         this.storeRecordVersions = storeRecordVersions;
         this.version = version;
@@ -342,6 +349,11 @@ public List<FormerIndex> getFormerIndexes() {
         return formerIndexes;
     }
 
+    @Nonnull
+    public Collection<ScalarValuedFunction> getAllScalarValuedFunctions() {
+        return scalarValuedFunctions;
+    }
+
     public boolean isSplitLongRecords() {
         return splitLongRecords;
     }
@@ -693,6 +705,7 @@ public RecordMetaDataProto.MetaData toProto(@Nullable Descriptors.FileDescriptor
         }
 
         // Add in the final options.
+        builder.addAllScalarValuedFunction(scalarValuedFunctions.stream().map(ScalarValuedFunction::toProto).collect(Collectors.toList()));
         builder.setSplitLongRecords(splitLongRecords);
         builder.setStoreRecordVersions(storeRecordVersions);
         builder.setVersion(version);

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/RecordMetaDataBuilder.java

@@ -35,6 +35,7 @@
 import com.apple.foundationdb.record.metadata.RecordTypeIndexesBuilder;
 import com.apple.foundationdb.record.metadata.SyntheticRecordType;
 import com.apple.foundationdb.record.metadata.SyntheticRecordTypeBuilder;
+import com.apple.foundationdb.record.metadata.ScalarValuedFunction;
 import com.apple.foundationdb.record.metadata.UnnestedRecordTypeBuilder;
 import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression;
 import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
@@ -110,6 +111,8 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider {
     @Nonnull
     private final Map<String, SyntheticRecordTypeBuilder<?>> syntheticRecordTypes;
     @Nonnull
+    private final Set<ScalarValuedFunction> scalarValuedFunctions;
+    @Nonnull
     private final Map<String, Index> indexes;
     @Nonnull
     private final Map<String, Index> universalIndexes;
@@ -144,6 +147,7 @@ public class RecordMetaDataBuilder implements RecordMetaDataProvider {
         indexMaintainerRegistry = IndexMaintainerRegistryImpl.instance();
         evolutionValidator = MetaDataEvolutionValidator.getDefaultInstance();
         syntheticRecordTypes = new HashMap<>();
+        scalarValuedFunctions = new HashSet<>();
     }
 
     private void processSchemaOptions(boolean processExtensionOptions) {
@@ -222,6 +226,9 @@ private void loadProtoExceptRecords(@Nonnull RecordMetaDataProto.MetaData metaDa
                 typeBuilder.setRecordTypeKey(LiteralKeyExpression.fromProtoValue(typeProto.getExplicitKey()));
             }
         }
+        for (RecordMetaDataProto.ScalarValuedFunction scalarValuedFunction: metaDataProto.getScalarValuedFunctionList()) {
+            scalarValuedFunctions.add(ScalarValuedFunction.fromProto(scalarValuedFunction));
+        }
         if (metaDataProto.hasSplitLongRecords()) {
             splitLongRecords = metaDataProto.getSplitLongRecords();
         }
@@ -1179,6 +1186,14 @@ public void addFormerIndex(@Nonnull FormerIndex formerIndex) {
         formerIndexes.add(formerIndex);
     }
 
+    public void addScalarValuedFunction(@Nonnull ScalarValuedFunction scalarValuedFunction) {
+        scalarValuedFunctions.add(scalarValuedFunction);
+    }
+
+    public void addScalarValuedFunctions(@Nonnull Iterable<? extends ScalarValuedFunction> scalarValuedFunctions) {
+        scalarValuedFunctions.forEach(this.scalarValuedFunctions::add);
+    }
+
     public boolean isSplitLongRecords() {
         return splitLongRecords;
     }
@@ -1420,7 +1435,7 @@ public RecordMetaData build(boolean validate) {
         Map<Object, SyntheticRecordType<?>> recordTypeKeyToSyntheticRecordTypeMap = Maps.newHashMapWithExpectedSize(syntheticRecordTypes.size());
         RecordMetaData metaData = new RecordMetaData(recordsDescriptor, getUnionDescriptor(), unionFields,
                 builtRecordTypes, builtSyntheticRecordTypes, recordTypeKeyToSyntheticRecordTypeMap,
-                indexes, universalIndexes, formerIndexes,
+                indexes, universalIndexes, formerIndexes, scalarValuedFunctions,
                 splitLongRecords, storeRecordVersions, version, subspaceKeyCounter, usesSubspaceKeyCounter, recordCountKey, localFileDescriptor != null);
         for (RecordTypeBuilder recordTypeBuilder : recordTypes.values()) {
             KeyExpression primaryKey = recordTypeBuilder.getPrimaryKey();

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/Key.java

@@ -22,7 +22,7 @@
 
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.metadata.expressions.EmptyKeyExpression;
 import com.apple.foundationdb.record.metadata.expressions.FieldKeyExpression;
@@ -392,21 +392,21 @@ public static class Evaluated {
          * Values used in index keys in place of missing fields.
          */
         public enum NullStandin {
-            NULL(RecordMetaDataProto.Field.NullInterpretation.NOT_UNIQUE), // Missing field here skips uniqueness checks.
-            NULL_UNIQUE(RecordMetaDataProto.Field.NullInterpretation.UNIQUE), // Missing field here like ordinary value, but null, for uniqueness.
-            NOT_NULL(RecordMetaDataProto.Field.NullInterpretation.NOT_NULL); // Missing field has type's ordinary default value.
+            NULL(RecordKeyExpressionProto.Field.NullInterpretation.NOT_UNIQUE), // Missing field here skips uniqueness checks.
+            NULL_UNIQUE(RecordKeyExpressionProto.Field.NullInterpretation.UNIQUE), // Missing field here like ordinary value, but null, for uniqueness.
+            NOT_NULL(RecordKeyExpressionProto.Field.NullInterpretation.NOT_NULL); // Missing field has type's ordinary default value.
 
-            private RecordMetaDataProto.Field.NullInterpretation proto;
+            private RecordKeyExpressionProto.Field.NullInterpretation proto;
 
-            NullStandin(RecordMetaDataProto.Field.NullInterpretation nullInterpretation) {
+            NullStandin(RecordKeyExpressionProto.Field.NullInterpretation nullInterpretation) {
                 proto = nullInterpretation;
             }
 
-            public RecordMetaDataProto.Field.NullInterpretation toProto() {
+            public RecordKeyExpressionProto.Field.NullInterpretation toProto() {
                 return proto;
             }
 
-            public static NullStandin valueOf(RecordMetaDataProto.Field.NullInterpretation nullInterpretation) throws KeyExpression.DeserializationException {
+            public static NullStandin valueOf(RecordKeyExpressionProto.Field.NullInterpretation nullInterpretation) throws KeyExpression.DeserializationException {
                 switch (nullInterpretation) {
                     case NOT_UNIQUE:
                         return NULL;

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/ScalarValuedFunction.java

@@ -0,0 +1,67 @@
+/*
+ * UDF.java
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2015-2025 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.apple.foundationdb.record.metadata;
+
+import com.apple.foundationdb.record.PlanHashable;
+import com.apple.foundationdb.record.PlanSerializationContext;
+import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.query.plan.cascades.MacroFunction;
+import com.apple.foundationdb.record.query.plan.serialization.DefaultPlanSerializationRegistry;
+
+import javax.annotation.Nonnull;
+
+/**
+ * Defines a scalar User-defined-function.
+ */
+public class ScalarValuedFunction {
+    @Nonnull
+    private final MacroFunction macroFunction;
+
+    public ScalarValuedFunction(@Nonnull MacroFunction functionValue) {
+        this.macroFunction = functionValue;
+    }
+
+    @Nonnull
+    public MacroFunction getMacroFunction() {
+        return macroFunction;
+    }
+
+    @Nonnull
+    public String getFunctionName() {
+        return macroFunction.getFunctionName();
+    }
+
+    @Nonnull
+    public RecordMetaDataProto.ScalarValuedFunction toProto() {
+        PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE,
+                PlanHashable.CURRENT_FOR_CONTINUATION);
+        return RecordMetaDataProto.ScalarValuedFunction.newBuilder()
+                .setFunctionValue(macroFunction.toProto(serializationContext))
+                .build();
+    }
+
+    @Nonnull
+    public static ScalarValuedFunction fromProto(RecordMetaDataProto.ScalarValuedFunction proto) {
+        PlanSerializationContext serializationContext = new PlanSerializationContext(DefaultPlanSerializationRegistry.INSTANCE,
+                PlanHashable.CURRENT_FOR_CONTINUATION);
+        return new ScalarValuedFunction(MacroFunction.fromProto(serializationContext, proto.getFunctionValue()));
+    }
+}

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/DimensionsKeyExpression.java

@@ -23,7 +23,7 @@
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.record.ObjectPlanHash;
 import com.apple.foundationdb.record.PlanHashable;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
 import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
@@ -63,7 +63,7 @@ private DimensionsKeyExpression(@Nonnull final KeyExpression wholeKey,
         this.dimensionsSize = dimensionsSize;
     }
 
-    DimensionsKeyExpression(@Nonnull final RecordMetaDataProto.Dimensions dimensions) throws DeserializationException {
+    DimensionsKeyExpression(@Nonnull final RecordKeyExpressionProto.Dimensions dimensions) throws DeserializationException {
         this(KeyExpression.fromProto(dimensions.getWholeKey()), dimensions.getPrefixSize(), dimensions.getDimensionsSize());
     }
 
@@ -114,8 +114,8 @@ public boolean needsCopyingToPartialRecord() {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.Dimensions toProto() throws SerializationException {
-        final RecordMetaDataProto.Dimensions.Builder builder = RecordMetaDataProto.Dimensions.newBuilder();
+    public RecordKeyExpressionProto.Dimensions toProto() throws SerializationException {
+        final RecordKeyExpressionProto.Dimensions.Builder builder = RecordKeyExpressionProto.Dimensions.newBuilder();
         builder.setWholeKey(getWholeKey().toKeyExpression());
         builder.setPrefixSize(prefixSize);
         builder.setDimensionsSize(dimensionsSize);
@@ -124,8 +124,8 @@ public RecordMetaDataProto.Dimensions toProto() throws SerializationException {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.KeyExpression toKeyExpression() {
-        return RecordMetaDataProto.KeyExpression.newBuilder().setDimensions(toProto()).build();
+    public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
+        return RecordKeyExpressionProto.KeyExpression.newBuilder().setDimensions(toProto()).build();
     }
 
     @Nonnull

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/EmptyKeyExpression.java

@@ -23,7 +23,7 @@
 import com.apple.foundationdb.annotation.API;
 import com.apple.foundationdb.record.ObjectPlanHash;
 import com.apple.foundationdb.record.PlanHashable;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
 import com.apple.foundationdb.record.query.plan.cascades.KeyExpressionVisitor;
@@ -45,8 +45,8 @@ public class EmptyKeyExpression extends BaseKeyExpression implements KeyExpressi
     private static final ObjectPlanHash BASE_HASH = new ObjectPlanHash("Empty-Key-Expression");
 
     public static final EmptyKeyExpression EMPTY = new EmptyKeyExpression();
-    public static final RecordMetaDataProto.KeyExpression EMPTY_PROTO =
-            RecordMetaDataProto.KeyExpression.newBuilder().setEmpty(EMPTY.toProto()).build();
+    public static final RecordKeyExpressionProto.KeyExpression EMPTY_PROTO =
+            RecordKeyExpressionProto.KeyExpression.newBuilder().setEmpty(EMPTY.toProto()).build();
 
     private EmptyKeyExpression() {
         // nothing to initialize
@@ -75,8 +75,8 @@ public int getColumnSize() {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.Empty toProto() throws SerializationException {
-        return RecordMetaDataProto.Empty.getDefaultInstance();
+    public RecordKeyExpressionProto.Empty toProto() throws SerializationException {
+        return RecordKeyExpressionProto.Empty.getDefaultInstance();
     }
 
     @Nonnull
@@ -87,7 +87,7 @@ public <S extends KeyExpressionVisitor.State, R> R expand(@Nonnull final KeyExpr
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.KeyExpression toKeyExpression() {
+    public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
         return EMPTY_PROTO;
     }
 

fdb-record-layer-core/src/main/java/com/apple/foundationdb/record/metadata/expressions/FieldKeyExpression.java

@@ -24,7 +24,7 @@
 import com.apple.foundationdb.record.ObjectPlanHash;
 import com.apple.foundationdb.record.PlanHashable;
 import com.apple.foundationdb.record.RecordCoreException;
-import com.apple.foundationdb.record.RecordMetaDataProto;
+import com.apple.foundationdb.record.expressions.RecordKeyExpressionProto;
 import com.apple.foundationdb.record.logging.LogMessageKeys;
 import com.apple.foundationdb.record.metadata.Key;
 import com.apple.foundationdb.record.provider.foundationdb.FDBRecord;
@@ -71,7 +71,7 @@ public FieldKeyExpression(@Nonnull String fieldName, @Nonnull FanType fanType, @
         this.nullStandin = nullStandin;
     }
 
-    public FieldKeyExpression(@Nonnull RecordMetaDataProto.Field field) throws DeserializationException {
+    public FieldKeyExpression(@Nonnull RecordKeyExpressionProto.Field field) throws DeserializationException {
         if (!field.hasFieldName()) {
             throw new DeserializationException("Serialized Field is missing field name");
         }
@@ -193,8 +193,8 @@ public int getColumnSize() {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.Field toProto() throws SerializationException {
-        return RecordMetaDataProto.Field.newBuilder()
+    public RecordKeyExpressionProto.Field toProto() throws SerializationException {
+        return RecordKeyExpressionProto.Field.newBuilder()
                 .setFieldName(fieldName)
                 .setFanType(fanType.toProto())
                 .setNullInterpretation(nullStandin.toProto())
@@ -203,8 +203,8 @@ public RecordMetaDataProto.Field toProto() throws SerializationException {
 
     @Nonnull
     @Override
-    public RecordMetaDataProto.KeyExpression toKeyExpression() {
-        return RecordMetaDataProto.KeyExpression.newBuilder().setField(toProto()).build();
+    public RecordKeyExpressionProto.KeyExpression toKeyExpression() {
+        return RecordKeyExpressionProto.KeyExpression.newBuilder().setField(toProto()).build();
     }
 
     @Nonnull