diff --git a/.gitignore b/.gitignore index 12971f9e1..00584490b 100644 --- a/.gitignore +++ b/.gitignore @@ -193,3 +193,4 @@ packages/ *.lib nativebin/ /cs/benchmark/Properties/launchSettings.json +/cs/playground/FasterPSFSample/Properties/launchSettings.json diff --git a/cs/FASTER.sln b/cs/FASTER.sln index 17c2dd5fe..df3a2847a 100644 --- a/cs/FASTER.sln +++ b/cs/FASTER.sln @@ -53,6 +53,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FasterLogPubSub", "samples\ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureBackedStore", "samples\AzureBackedStore\AzureBackedStore.csproj", "{E2A1C205-4D35-448C-A72F-B9A4AE28EB4E}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FasterPSFSample", "playground\FasterPSFSample\FasterPSFSample.csproj", "{BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -205,6 +207,52 @@ Global {E2A1C205-4D35-448C-A72F-B9A4AE28EB4E}.Release|Any CPU.Build.0 = Release|x64 {E2A1C205-4D35-448C-A72F-B9A4AE28EB4E}.Release|x64.ActiveCfg = Release|x64 {E2A1C205-4D35-448C-A72F-B9A4AE28EB4E}.Release|x64.Build.0 = Release|x64 + {25C5C6B6-4A8A-46DD-88C1-EB247033FE58}.Debug|Any CPU.ActiveCfg = Debug|x64 + {25C5C6B6-4A8A-46DD-88C1-EB247033FE58}.Debug|Any CPU.Build.0 = Debug|x64 + {25C5C6B6-4A8A-46DD-88C1-EB247033FE58}.Debug|x64.ActiveCfg = Debug|x64 + {25C5C6B6-4A8A-46DD-88C1-EB247033FE58}.Debug|x64.Build.0 = Debug|x64 + {25C5C6B6-4A8A-46DD-88C1-EB247033FE58}.Release|Any CPU.ActiveCfg = Release|x64 + {25C5C6B6-4A8A-46DD-88C1-EB247033FE58}.Release|Any CPU.Build.0 = Release|x64 + {25C5C6B6-4A8A-46DD-88C1-EB247033FE58}.Release|x64.ActiveCfg = Release|x64 + {25C5C6B6-4A8A-46DD-88C1-EB247033FE58}.Release|x64.Build.0 = Release|x64 + {859F76F4-93D8-4D60-BF9A-363E217FA247}.Debug|Any CPU.ActiveCfg = Debug|x64 + {859F76F4-93D8-4D60-BF9A-363E217FA247}.Debug|Any CPU.Build.0 = Debug|x64 + {859F76F4-93D8-4D60-BF9A-363E217FA247}.Debug|x64.ActiveCfg = Debug|x64 + {859F76F4-93D8-4D60-BF9A-363E217FA247}.Debug|x64.Build.0 = Debug|x64 + {859F76F4-93D8-4D60-BF9A-363E217FA247}.Release|Any CPU.ActiveCfg = Release|x64 + {859F76F4-93D8-4D60-BF9A-363E217FA247}.Release|Any CPU.Build.0 = Release|x64 + {859F76F4-93D8-4D60-BF9A-363E217FA247}.Release|x64.ActiveCfg = Release|x64 + {859F76F4-93D8-4D60-BF9A-363E217FA247}.Release|x64.Build.0 = Release|x64 + {95AC8766-84F9-4E95-B2E9-2169B6375FB2}.Debug|Any CPU.ActiveCfg = Debug|x64 + {95AC8766-84F9-4E95-B2E9-2169B6375FB2}.Debug|Any CPU.Build.0 = Debug|x64 + {95AC8766-84F9-4E95-B2E9-2169B6375FB2}.Debug|x64.ActiveCfg = Debug|x64 + {95AC8766-84F9-4E95-B2E9-2169B6375FB2}.Debug|x64.Build.0 = Debug|x64 + {95AC8766-84F9-4E95-B2E9-2169B6375FB2}.Release|Any CPU.ActiveCfg = Release|x64 + {95AC8766-84F9-4E95-B2E9-2169B6375FB2}.Release|Any CPU.Build.0 = Release|x64 + {95AC8766-84F9-4E95-B2E9-2169B6375FB2}.Release|x64.ActiveCfg = Release|x64 + {95AC8766-84F9-4E95-B2E9-2169B6375FB2}.Release|x64.Build.0 = Release|x64 + {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49}.Debug|Any CPU.ActiveCfg = Debug|x64 + {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49}.Debug|Any CPU.Build.0 = Debug|x64 + {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49}.Debug|x64.ActiveCfg = Debug|x64 + {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49}.Debug|x64.Build.0 = Debug|x64 + {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49}.Release|Any CPU.ActiveCfg = Release|x64 + {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49}.Release|Any CPU.Build.0 = Release|x64 + {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49}.Release|x64.ActiveCfg = Release|x64 + {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49}.Release|x64.Build.0 = Release|x64 + {340E0337-BB3B-4328-BDB7-E6FA63C63EA1}.Debug|Any CPU.ActiveCfg = Debug|x64 + {340E0337-BB3B-4328-BDB7-E6FA63C63EA1}.Debug|x64.ActiveCfg = Debug|x64 + {340E0337-BB3B-4328-BDB7-E6FA63C63EA1}.Debug|x64.Build.0 = Debug|x64 + {340E0337-BB3B-4328-BDB7-E6FA63C63EA1}.Release|Any CPU.ActiveCfg = Release|x64 + {340E0337-BB3B-4328-BDB7-E6FA63C63EA1}.Release|x64.ActiveCfg = Release|x64 + {340E0337-BB3B-4328-BDB7-E6FA63C63EA1}.Release|x64.Build.0 = Release|x64 + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Debug|x64.ActiveCfg = Debug|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Debug|x64.Build.0 = Debug|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Release|Any CPU.Build.0 = Release|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Release|x64.ActiveCfg = Release|Any CPU + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -230,6 +278,12 @@ Global {DACB12EB-8A64-4BB4-BFA3-0377AACD28D3} = {62BC1134-B6E1-476A-B894-7CA278A8B6DE} {F6EA46D5-DD66-47F2-8FAC-370FDD733DD3} = {62BC1134-B6E1-476A-B894-7CA278A8B6DE} {E2A1C205-4D35-448C-A72F-B9A4AE28EB4E} = {62BC1134-B6E1-476A-B894-7CA278A8B6DE} + {25C5C6B6-4A8A-46DD-88C1-EB247033FE58} = {E6026D6A-01C5-4582-B2C1-64751490DABE} + {859F76F4-93D8-4D60-BF9A-363E217FA247} = {E6026D6A-01C5-4582-B2C1-64751490DABE} + {95AC8766-84F9-4E95-B2E9-2169B6375FB2} = {E6026D6A-01C5-4582-B2C1-64751490DABE} + {642DCE86-1BAA-4FFF-98BF-0FB9BB11CD49} = {E6026D6A-01C5-4582-B2C1-64751490DABE} + {340E0337-BB3B-4328-BDB7-E6FA63C63EA1} = {E6026D6A-01C5-4582-B2C1-64751490DABE} + {BBC2B5E3-4D3E-49FE-BE23-99F859E0D386} = {E6026D6A-01C5-4582-B2C1-64751490DABE} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A0750637-2CCB-4139-B25E-F2CE740DCFAC} diff --git a/cs/playground/FasterPSFSample/BlittableOrders.cs b/cs/playground/FasterPSFSample/BlittableOrders.cs new file mode 100644 index 000000000..c4111a3b7 --- /dev/null +++ b/cs/playground/FasterPSFSample/BlittableOrders.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System; + +namespace FasterPSFSample +{ + public struct BlittableOrders : IOrders + { + public int Id { get; set; } + + // Colors, strings, and enums are not blittable so we use int + public int SizeInt { get; set; } + + public int ColorArgb { get; set; } + + public int Count { get; set; } + + public override string ToString() => $"{(Constants.Size)this.SizeInt}, {Constants.ColorDict[this.ColorArgb].Name}, {Count}"; + + public class Functions : IFunctions, Output, Context> + { + #region Read + public void ConcurrentReader(ref Key key, ref Input input, ref BlittableOrders value, ref Output dst) + => dst.Value = value; + + public void SingleReader(ref Key key, ref Input input, ref BlittableOrders value, ref Output dst) + => dst.Value = value; + + public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) + { /* Output is not set by pending operations */ } + #endregion Read + + #region Upsert + public bool ConcurrentWriter(ref Key key, ref BlittableOrders src, ref BlittableOrders dst) + { + dst = src; + return true; + } + + public void SingleWriter(ref Key key, ref BlittableOrders src, ref BlittableOrders dst) + => dst = src; + + public void UpsertCompletionCallback(ref Key key, ref BlittableOrders value, Context context) + { } + #endregion Upsert + + #region RMW + public void CopyUpdater(ref Key key, ref Input input, ref BlittableOrders oldValue, ref BlittableOrders newValue) + => throw new NotImplementedException(); + + public void InitialUpdater(ref Key key, ref Input input, ref BlittableOrders value) + => value = input.InitialUpdateValue; + + public bool InPlaceUpdater(ref Key key, ref Input input, ref BlittableOrders value) + { + value.ColorArgb = input.IPUColorInt; + return true; + } + + public void RMWCompletionCallback(ref Key key, ref Input input, Context context, Status status) + { } + #endregion RMW + + public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) + { } + + public void DeleteCompletionCallback(ref Key key, Context context) + { } + } + } +} diff --git a/cs/playground/FasterPSFSample/ColorKey.cs b/cs/playground/FasterPSFSample/ColorKey.cs new file mode 100644 index 000000000..7420fba8f --- /dev/null +++ b/cs/playground/FasterPSFSample/ColorKey.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System.Drawing; + +namespace FasterPSFSample +{ + public struct ColorKey : IFasterEqualityComparer + { + // Colors, strings, and enums are not blittable so we use int + public int ColorArgb; + + public ColorKey(Color color) => this.ColorArgb = color.ToArgb(); + + public override string ToString() => Constants.ColorDict[this.ColorArgb].Name; + + public long GetHashCode64(ref ColorKey key) => Utility.GetHashCode(key.ColorArgb); + + public bool Equals(ref ColorKey k1, ref ColorKey k2) => k1.ColorArgb == k2.ColorArgb; + } +} diff --git a/cs/playground/FasterPSFSample/CombinedKey.cs b/cs/playground/FasterPSFSample/CombinedKey.cs new file mode 100644 index 000000000..8d93de87b --- /dev/null +++ b/cs/playground/FasterPSFSample/CombinedKey.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System.Drawing; + +namespace FasterPSFSample +{ + public struct CombinedKey : IFasterEqualityComparer + { + public int ValueType; + public int ValueInt; + + public CombinedKey(Constants.Size size) + { + this.ValueType = (int)Constants.ValueType.Size; + this.ValueInt = (int)size; + } + + public CombinedKey(Color color) + { + this.ValueType = (int)Constants.ValueType.Color; + this.ValueInt = color.ToArgb(); + } + + public CombinedKey(int countBin) + { + this.ValueType = (int)Constants.ValueType.Count; + this.ValueInt = countBin; + } + + public override string ToString() + { + var valueType = (Constants.ValueType)this.ValueType; + var valueTypeString = $"{valueType}"; + return valueType switch + { + Constants.ValueType.Size => $"{valueTypeString}: {(Constants.Size)this.ValueInt}", + Constants.ValueType.Color => $"{valueTypeString}: {Constants.ColorDict[this.ValueInt]}", + Constants.ValueType.Count => $"{valueTypeString}: {this.ValueInt}", + _ => throw new System.NotImplementedException("Unknown ValueType") + }; + } + + public long GetHashCode64(ref CombinedKey key) + => (long)key.ValueType << 32 | (uint)key.ValueInt.GetHashCode(); + + public bool Equals(ref CombinedKey k1, ref CombinedKey k2) + => k1.ValueType == k2.ValueType && k1.ValueInt == k2.ValueInt; + } +} diff --git a/cs/playground/FasterPSFSample/Constants.cs b/cs/playground/FasterPSFSample/Constants.cs new file mode 100644 index 000000000..2483552e5 --- /dev/null +++ b/cs/playground/FasterPSFSample/Constants.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Drawing; + +namespace FasterPSFSample +{ + public static class Constants + { + // Colors, strings, and enums are not blittable so we store int + public enum Size + { + Small, + Medium, + Large, + XLarge, + XXLarge, + NumSizes + } + + static internal Dictionary ColorDict = new Dictionary + { + [Color.Black.ToArgb()] = Color.Black, + [Color.Red.ToArgb()] = Color.Red, + [Color.Green.ToArgb()] = Color.Green, + [Color.Blue.ToArgb()] = Color.Blue, + [Color.Purple.ToArgb()] = Color.Purple + }; + + static internal Color[] Colors = { Color.Black, Color.Red, Color.Green, Color.Blue, Color.Purple }; + + public enum ValueType + { + Size, + Color, + Count + } + } +} diff --git a/cs/playground/FasterPSFSample/Context.cs b/cs/playground/FasterPSFSample/Context.cs new file mode 100644 index 000000000..a0e72f679 --- /dev/null +++ b/cs/playground/FasterPSFSample/Context.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Collections.Generic; + +namespace FasterPSFSample +{ + public class Context + { + public List Value { get; set; } = new List(); + } +} diff --git a/cs/playground/FasterPSFSample/CountBinKey.cs b/cs/playground/FasterPSFSample/CountBinKey.cs new file mode 100644 index 000000000..2271b04e2 --- /dev/null +++ b/cs/playground/FasterPSFSample/CountBinKey.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; + +namespace FasterPSFSample +{ + public struct CountBinKey : IFasterEqualityComparer + { + internal const int BinSize = 100; + internal const int MaxOrders = BinSize * 10; + internal const int LastBin = 9; // 0-based + internal static bool WantLastBin; + + public int Bin; + + public CountBinKey(int bin) => this.Bin = bin; + + internal static bool GetBin(int numOrders, out int bin) + { + // Skip the last bin during initial inserts to illustrate not matching the PSF (returning null) + bin = numOrders / BinSize; + return WantLastBin || bin < LastBin; + } + + // Make the hashcode for this distinct from size enum values + public long GetHashCode64(ref CountBinKey key) => Utility.GetHashCode(key.Bin + 1000); + + public bool Equals(ref CountBinKey k1, ref CountBinKey k2) => k1.Bin == k2.Bin; + } +} diff --git a/cs/playground/FasterPSFSample/FPSF.cs b/cs/playground/FasterPSFSample/FPSF.cs new file mode 100644 index 000000000..cc14a2498 --- /dev/null +++ b/cs/playground/FasterPSFSample/FPSF.cs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FasterPSFSample +{ + class FPSF + where TValue : IOrders + where TFunctions : IFunctions> + where TSerializer : BinaryObjectSerializer, new() + { + internal IFasterKV FasterKV { get; set; } + + private LogFiles logFiles; + + // MultiGroup PSFs -- different key types, one per group. + internal IPSF SizePsf, ColorPsf, CountBinPsf; + internal IPSF CombinedSizePsf, CombinedColorPsf, CombinedCountBinPsf; + + internal FPSF(bool useObjectValues, bool useMultiGroup, bool useReadCache) + { + this.logFiles = new LogFiles(useObjectValues, useReadCache, useMultiGroup ? 3 : 1); + + this.FasterKV = new FasterKV( + 1L << 20, this.logFiles.LogSettings, + null, // TODO: add checkpoints + useObjectValues ? new SerializerSettings { valueSerializer = () => new TSerializer() } : null); + + if (useMultiGroup) + { + var groupOrdinal = 0; + this.SizePsf = FasterKV.RegisterPSF(CreatePSFRegistrationSettings(groupOrdinal++), nameof(this.SizePsf), + (k, v) => new SizeKey((Constants.Size)v.SizeInt)); + this.ColorPsf = FasterKV.RegisterPSF(CreatePSFRegistrationSettings(groupOrdinal++), nameof(this.ColorPsf), + (k, v) => new ColorKey(Constants.ColorDict[v.ColorArgb])); + this.CountBinPsf = FasterKV.RegisterPSF(CreatePSFRegistrationSettings(groupOrdinal++), nameof(this.CountBinPsf), + (k, v) => CountBinKey.GetBin(v.Count, out int bin) ? new CountBinKey(bin) : (CountBinKey?)null); + } + else + { + var psfs = FasterKV.RegisterPSF(CreatePSFRegistrationSettings(0), + new (string, Func)[] + { + (nameof(this.SizePsf), (k, v) => new CombinedKey((Constants.Size)v.SizeInt)), + (nameof(this.ColorPsf), (k, v) => new CombinedKey(Constants.ColorDict[v.ColorArgb])), + (nameof(this.CountBinPsf), (k, v) => CountBinKey.GetBin(v.Count, out int bin) + ? new CombinedKey(bin) : (CombinedKey?)null) + }); + this.CombinedSizePsf = psfs[0]; + this.CombinedColorPsf = psfs[1]; + this.CombinedCountBinPsf = psfs[2]; + } + } + + PSFRegistrationSettings CreatePSFRegistrationSettings(int groupOrdinal) + { + var regSettings = new PSFRegistrationSettings + { + HashTableSize = 1L << 20, + LogSettings = this.logFiles.PSFLogSettings[groupOrdinal], + CheckpointSettings = new CheckpointSettings(), // TODO checkpoints + IPU1CacheSize = 0, // TODO IPUCache + IPU2CacheSize = 0 + }; + + // Override some things. + var regLogSettings = regSettings.LogSettings; + regLogSettings.PageSizeBits = 20; + regLogSettings.SegmentSizeBits = 25; + regLogSettings.MemorySizeBits = 29; + regLogSettings.CopyReadsToTail = false; // TODO--test this in both primary and secondary FKV + if (!(regLogSettings.ReadCacheSettings is null)) + { + regLogSettings.ReadCacheSettings.PageSizeBits = regLogSettings.PageSizeBits; + regLogSettings.ReadCacheSettings.MemorySizeBits = regLogSettings.MemorySizeBits; + } + return regSettings; + } + + internal void Close() + { + if (!(this.FasterKV is null)) + { + this.FasterKV.Dispose(); + this.FasterKV = null; + } + if (!(this.logFiles is null)) + { + this.logFiles.Close(); + this.logFiles = null; + } + } + } +} diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.cs b/cs/playground/FasterPSFSample/FasterPSFSample.cs new file mode 100644 index 000000000..8cc414245 --- /dev/null +++ b/cs/playground/FasterPSFSample/FasterPSFSample.cs @@ -0,0 +1,666 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +//#define PSF_TRACE + +using FASTER.core; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Drawing; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace FasterPSFSample +{ + public partial class FasterPSFSample + { + private const int UpsertCount = 100; + private static int blueCount, mediumCount, bin7Count; + private static int intersectMediumBlueCount, unionMediumBlueCount; + private static int intersectMediumBlue7Count, unionMediumBlue7Count; + private static int unionMediumLargeCount, unionRedBlueCount; + private static int unionMediumLargeRedBlueCount, intersectMediumLargeRedBlueCount; + + internal static Dictionary keyDict = new Dictionary(); + + internal static HashSet lastBinKeys = new HashSet(); + private static int initialSkippedLastBinCount; + + private static int nextId = 1000000000; + + internal static int serialNo; + + static async Task Main(string[] argv) + { + if (!ParseArgs(argv)) + return; + + if (useObjectValue) // TODO add VarLenValue + await RunSample, Output, ObjectOrders.Functions, ObjectOrders.Serializer>(); + else + await RunSample, Output, BlittableOrders.Functions, NoSerializer>(); + return; + } + + internal async static Task RunSample() + where TValue : IOrders, new() + where TInput : IInput, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + var fpsf = new FPSF(useObjectValue, useMultiGroups, useReadCache: false); // ReadCache and CopyReadsToTail are not supported for PSFs + try + { + CountBinKey.WantLastBin = false; + await RunInitialInserts(fpsf); + CountBinKey.WantLastBin = true; + await RunReads(fpsf); + var ok = await QueryPSFsWithoutBoolOps(fpsf) + && await QueryPSFsWithBoolOps(fpsf) + && await UpdateSizeByUpsert(fpsf) + && await UpdateColorByRMW(fpsf) + && await UpdateCountByUpsert(fpsf) + && await Delete(fpsf); + Console.WriteLine("--------------------------------------------------------"); + Console.WriteLine($"Completed run: UseObjects {useObjectValue}, MultiGroup {useMultiGroups}, Async {useAsync}"); + Console.WriteLine(); + Console.Write("===>>> "); + Console.WriteLine(ok ? "Passed! All operations succeeded" : "*** Failed! *** One or more operations failed"); + Console.WriteLine(); + if (Debugger.IsAttached) + { + Console.Write("Press ENTER to close this window . . ."); + Console.ReadLine(); + } + } + finally + { + fpsf.Close(); + } + + //Console.WriteLine("Press to end"); + //Console.ReadLine(); + } + + [Conditional("PSF_TRACE")] static void PsfTrace(string message) => Console.Write(message); + + [Conditional("PSF_TRACE")] static void PsfTraceLine(string message) => Console.WriteLine(message); + + internal async static Task RunInitialInserts(FPSF fpsf) + where TValue : IOrders, new() + where TInput : IInput, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine("Writing keys from 0 to {0} to FASTER", UpsertCount); + + var rng = new Random(13); + using var session = fpsf.FasterKV.NewSession, TFunctions>(new TFunctions()); + var context = new Context(); + var input = default(TInput); + + for (int ii = 0; ii < UpsertCount; ii++) + { + // Leave the last value unassigned from each category (we'll use it to update later) + var key = new Key(Interlocked.Increment(ref nextId) - 1); + var value = new TValue + { + Id = key.Id, + SizeInt = rng.Next((int)Constants.Size.NumSizes - 1), + ColorArgb = Constants.Colors[rng.Next(Constants.Colors.Length - 1)].ToArgb(), + Count = rng.Next(CountBinKey.MaxOrders - 1) + }; + + keyDict[key] = value; + CountBinKey.GetBin(value.Count, out int bin); + var isBlue = value.ColorArgb == Color.Blue.ToArgb(); + var isRed = value.ColorArgb == Color.Red.ToArgb(); + var isMedium = value.SizeInt == (int)Constants.Size.Medium; + var isLarge = value.SizeInt == (int)Constants.Size.Large; + var isBin7 = bin == 7; + + if (isBlue) + { + ++blueCount; + if (isMedium) + { + ++intersectMediumBlueCount; + if (isBin7) + ++intersectMediumBlue7Count; + } + } + if (isMedium) + ++mediumCount; + + if (isBin7) + ++bin7Count; + else if (bin == CountBinKey.LastBin) + lastBinKeys.Add(key); + + if (isMedium || isBlue) + ++unionMediumBlueCount; + if (isMedium || isBlue || isBin7) + ++unionMediumBlue7Count; + + var isMediumOrLarge = isMedium || isLarge; + var isRedOrBlue = isBlue || isRed; + + if (isMediumOrLarge) + { + ++unionMediumLargeCount; + if (isRedOrBlue) + ++intersectMediumLargeRedBlueCount; + } + if (isRedOrBlue) + ++unionRedBlueCount; + if (isMediumOrLarge || isRedOrBlue) + ++unionMediumLargeRedBlueCount; + + PsfTrace($"{value} |"); + + // Both Upsert and RMW do an insert when the key is not found. + if ((ii & 1) == 0) + { + if (useAsync) + await session.UpsertAsync(ref key, ref value, context); + else + session.Upsert(ref key, ref value, context, serialNo); + } + else + { + input.InitialUpdateValue = value; + if (useAsync) + await session.RMWAsync(ref key, ref input, context); + else + session.RMW(ref key, ref input, context, serialNo); + } + } + ++serialNo; + + initialSkippedLastBinCount = lastBinKeys.Count(); + Console.WriteLine($"Upserted {UpsertCount} elements"); + } + + private static void RemoveIfSkippedLastBinKey(ref Key key) + { + // TODO: If we can do IPU in PSFs rather than RCU, we'll need to see what was actually updated. + if (!useMultiGroups) + lastBinKeys.Remove(key); + } + + internal async static Task RunReads(FPSF fpsf) + where TValue : IOrders, new() + where TInput : IInput, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine("Reading {0} random keys from FASTER", UpsertCount); + + var rng = new Random(0); + int statusPending = 0; + var output = new TOutput(); + var input = default(TInput); + var context = new Context(); + var readCount = UpsertCount * 2; + + var keys = keyDict.Keys.ToArray(); + + using var session = fpsf.FasterKV.NewSession, TFunctions>(new TFunctions()); + for (int i = 0; i < UpsertCount; i++) + { + var key = keys[rng.Next(keys.Length)]; + var status = Status.OK; + if (useAsync) + { + (status, output) = (await session.ReadAsync(ref key, ref input, context)).CompleteRead(); + } + else + { + session.Read(ref key, ref input, ref output, context, serialNo); + } + + if (status == Status.OK && output.Value.MemberTuple != key.MemberTuple) + throw new Exception($"Error: Value does not match key in {nameof(RunReads)}"); + } + ++serialNo; + + session.CompletePending(true); + Console.WriteLine($"Read {readCount} elements with {++statusPending} Pending"); + } + + const string indent2 = " "; + const string indent4 = " "; + + static bool VerifyProviderDatas(FasterKVProviderData[] providerDatas, string name, int expectedCount) + where TValue : IOrders, new() + { + Console.Write($"{indent4}{name}: "); + if (verbose) + { + foreach (var providerData in providerDatas) + { + ref TValue value = ref providerData.GetValue(); + Console.WriteLine(indent4 + value); + } + } + Console.WriteLine(providerDatas.Length == expectedCount + ? $"Passed: expected == actual ({expectedCount})" + : $"Failed: expected ({expectedCount}) != actual ({providerDatas.Length})"); + return expectedCount == providerDatas.Length; + } + + internal async static Task QueryPSFsWithoutBoolOps(FPSF fpsf) + where TValue : IOrders, new() + where TInput : IInput, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine(); + Console.WriteLine("Querying PSFs from FASTER with no boolean ops", UpsertCount); + + using var session = fpsf.FasterKV.NewSession, TFunctions>(new TFunctions()); + FasterKVProviderData[] providerDatas = null; + var ok = true; + + async Task[]> RunQuery(IPSF psf, TPSFKey key) where TPSFKey : struct + => useAsync + ? await session.QueryPSFAsync(psf, key).ToArrayAsync() + : session.QueryPSF(psf, key).ToArray(); + + providerDatas = useMultiGroups + ? await RunQuery(fpsf.SizePsf, new SizeKey(Constants.Size.Medium)) + : await RunQuery(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium)); + ok &= VerifyProviderDatas(providerDatas, "Medium", mediumCount); + + providerDatas = useMultiGroups + ? await RunQuery(fpsf.ColorPsf, new ColorKey(Color.Blue)) + : await RunQuery(fpsf.CombinedColorPsf, new CombinedKey(Color.Blue)); + ok &= VerifyProviderDatas(providerDatas, "Blue", blueCount); + + providerDatas = useMultiGroups + ? await RunQuery(fpsf.CountBinPsf, new CountBinKey(7)) + : await RunQuery(fpsf.CombinedCountBinPsf, new CombinedKey(7)); + ok &= VerifyProviderDatas(providerDatas, "Bin7", bin7Count); + + providerDatas = useMultiGroups + ? await RunQuery(fpsf.CountBinPsf, new CountBinKey(CountBinKey.LastBin)) + : await RunQuery(fpsf.CombinedCountBinPsf, new CombinedKey(CountBinKey.LastBin)); + ok &= VerifyProviderDatas(providerDatas, "LastBin", 0); // Insert skipped (returned null from the PSF) all that fall into the last bin + return ok; + } + + internal async static Task QueryPSFsWithBoolOps(FPSF fpsf) + where TValue : IOrders, new() + where TInput : IInput, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine(); + Console.WriteLine("Querying PSFs from FASTER with boolean ops", UpsertCount); + + using var session = fpsf.FasterKV.NewSession, TFunctions>(new TFunctions()); + FasterKVProviderData[] providerDatas = null; + var ok = true; + + // Local functions can't be overloaded so make the name unique + async Task[]> RunQuery2( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + Func matchPredicate) + where TPSFKey1 : struct + where TPSFKey2 : struct + => useAsync + ? await session.QueryPSFAsync(psf1, key1, psf2, key2, matchPredicate).ToArrayAsync() + : session.QueryPSF(psf1, key1, psf2, key2, matchPredicate).ToArray(); + + async Task[]> RunQuery3( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + IPSF psf3, TPSFKey3 key3, + Func matchPredicate) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + => useAsync + ? await session.QueryPSFAsync(psf1, key1, psf2, key2, psf3, key3, matchPredicate).ToArrayAsync() + : session.QueryPSF(psf1, key1, psf2, key2, psf3, key3, matchPredicate).ToArray(); + + async Task[]> RunQuery1EnumKeys( + IPSF psf, IEnumerable keys, PSFQuerySettings querySettings = null) + where TPSFKey : struct + => useAsync + ? await session.QueryPSFAsync(psf, keys).ToArrayAsync() + : session.QueryPSF(psf, keys).ToArray(); + + async Task[]> RunQuery2Vec( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + Func matchPredicate) + where TPSFKey1 : struct + where TPSFKey2 : struct + => useAsync + ? await session.QueryPSFAsync(psf1, keys1, psf2, keys2, matchPredicate).ToArrayAsync() + : session.QueryPSF(psf1, keys1, psf2, keys2, matchPredicate).ToArray(); + + async Task[]> RunQuery1EnumTuple( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, + Func matchPredicate) + where TPSFKey : struct + => useAsync + ? await session.QueryPSFAsync(psfsAndKeys, matchPredicate).ToArrayAsync() + : session.QueryPSF(psfsAndKeys, matchPredicate).ToArray(); + + if (useMultiGroups) + { + providerDatas = await RunQuery2(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), + fpsf.ColorPsf, new ColorKey(Color.Blue), (sz, cl) => sz && cl); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlueCount), intersectMediumBlueCount); + providerDatas = await RunQuery2(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), + fpsf.ColorPsf, new ColorKey(Color.Blue), (sz, cl) => sz || cl); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlueCount), unionMediumBlueCount); + + providerDatas = await RunQuery3(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), + fpsf.ColorPsf, new ColorKey(Color.Blue), + fpsf.CountBinPsf, new CountBinKey(7), (sz, cl, ct) => sz && cl && ct); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlue7Count), intersectMediumBlue7Count); + providerDatas = await RunQuery3(fpsf.SizePsf, new SizeKey(Constants.Size.Medium), + fpsf.ColorPsf, new ColorKey(Color.Blue), + fpsf.CountBinPsf, new CountBinKey(7), (sz, cl, ct) => sz || cl || ct); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlue7Count), unionMediumBlue7Count); + + providerDatas = await RunQuery1EnumKeys(fpsf.SizePsf, new[] { new SizeKey(Constants.Size.Medium), new SizeKey(Constants.Size.Large) }); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeCount), unionMediumLargeCount); + providerDatas = await RunQuery1EnumKeys(fpsf.ColorPsf, new[] { new ColorKey(Color.Blue), new ColorKey(Color.Red) }); + ok &= VerifyProviderDatas(providerDatas, nameof(unionRedBlueCount), unionRedBlueCount); + + providerDatas = await RunQuery2Vec(fpsf.SizePsf, new[] { new SizeKey(Constants.Size.Medium), new SizeKey(Constants.Size.Large) }, + fpsf.ColorPsf, new[] { new ColorKey(Color.Blue), new ColorKey(Color.Red) }, (sz, cl) => sz && cl); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumLargeRedBlueCount), intersectMediumLargeRedBlueCount); + providerDatas = await RunQuery2Vec(fpsf.SizePsf, new[] { new SizeKey(Constants.Size.Medium), new SizeKey(Constants.Size.Large) }, + fpsf.ColorPsf, new[] { new ColorKey(Color.Blue), new ColorKey(Color.Red) }, (sz, cl) => sz || cl); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeRedBlueCount), unionMediumLargeRedBlueCount); + } + else + { + providerDatas = await RunQuery2(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), + fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), (sz, cl) => sz && cl); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlueCount), intersectMediumBlueCount); + providerDatas = await RunQuery1EnumTuple(new[] { (fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue) }.AsEnumerable()) }, sz => sz[0] && sz[1]); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlueCount), intersectMediumBlueCount); + // --- + providerDatas = await RunQuery2(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), + fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), (sz, cl) => sz || cl); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlueCount), unionMediumBlueCount); + providerDatas = await RunQuery1EnumTuple(new[] { (fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue) }.AsEnumerable()) }, sz => sz[0] || sz[1]); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlueCount), unionMediumBlueCount); + // --- + providerDatas = await RunQuery3(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), + fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), + fpsf.CombinedCountBinPsf, new CombinedKey(7), (sz, cl, ct) => sz && cl && ct); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlue7Count), intersectMediumBlue7Count); + providerDatas = await RunQuery1EnumTuple(new[] {(fpsf.CombinedSizePsf, new [] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new [] { new CombinedKey(Color.Blue) }.AsEnumerable()), + (fpsf.CombinedCountBinPsf, new [] { new CombinedKey(7) }.AsEnumerable()) }, sz => sz[0] && sz[1] && sz[2]); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumBlue7Count), intersectMediumBlue7Count); + // --- + providerDatas = await RunQuery3(fpsf.CombinedSizePsf, new CombinedKey(Constants.Size.Medium), + fpsf.CombinedColorPsf, new CombinedKey(Color.Blue), + fpsf.CombinedCountBinPsf, new CombinedKey(7), (sz, cl, ct) => sz || cl || ct); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlue7Count), unionMediumBlue7Count); + providerDatas = await RunQuery1EnumTuple(new[] {(fpsf.CombinedSizePsf, new [] { new CombinedKey(Constants.Size.Medium) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new [] { new CombinedKey(Color.Blue) }.AsEnumerable()), + (fpsf.CombinedCountBinPsf, new [] { new CombinedKey(7) }.AsEnumerable()) }, sz => sz[0] || sz[1] || sz[2]); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumBlue7Count), unionMediumBlue7Count); + // --- + providerDatas = await RunQuery1EnumKeys(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeCount), unionMediumLargeCount); + providerDatas = await RunQuery1EnumKeys(fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }); + ok &= VerifyProviderDatas(providerDatas, nameof(unionRedBlueCount), unionRedBlueCount); + // --- + providerDatas = await RunQuery2Vec(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }, + fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }, (sz, cl) => sz && cl); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumLargeRedBlueCount), intersectMediumLargeRedBlueCount); + providerDatas = await RunQuery1EnumTuple(new[] {(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }.AsEnumerable())}, sz => sz[0] && sz[1]); + ok &= VerifyProviderDatas(providerDatas, nameof(intersectMediumLargeRedBlueCount), intersectMediumLargeRedBlueCount); + // --- + providerDatas = await RunQuery2Vec(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }, + fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }, (sz, cl) => sz || cl); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeRedBlueCount), unionMediumLargeRedBlueCount); + providerDatas = await RunQuery1EnumTuple(new[] {(fpsf.CombinedSizePsf, new[] { new CombinedKey(Constants.Size.Medium), new CombinedKey(Constants.Size.Large) }.AsEnumerable()), + (fpsf.CombinedColorPsf, new[] { new CombinedKey(Color.Blue), new CombinedKey(Color.Red) }.AsEnumerable())}, sz => sz[0] || sz[1]); + ok &= VerifyProviderDatas(providerDatas, nameof(unionMediumLargeRedBlueCount), unionMediumLargeRedBlueCount); + } + + return ok; + } + + private static void WriteResult(bool isInitial, string name, int expectedCount, int actualCount) + { + var tag = isInitial ? "Initial" : "Updated"; + Console.WriteLine(expectedCount == actualCount + ? $"{indent4}{tag} {name} Passed: expected == actual ({expectedCount})" + : $"{indent4}{tag} {name} Failed: expected ({expectedCount}) != actual ({actualCount})"); + } + + internal async static Task UpdateSizeByUpsert(FPSF fpsf) + where TValue : IOrders, new() + where TInput : IInput, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine(); + Console.WriteLine("Updating Sizes via Upsert"); + + using var session = fpsf.FasterKV.NewSession, TFunctions>(new TFunctions()); + + FasterKVProviderData[] GetSizeDatas(Constants.Size size) + => useMultiGroups + ? session.QueryPSF(fpsf.SizePsf, new SizeKey(size)).ToArray() + : session.QueryPSF(fpsf.CombinedSizePsf, new CombinedKey(size)).ToArray(); + + var xxlDatas = GetSizeDatas(Constants.Size.XXLarge); + WriteResult(isInitial: true, "XXLarge", 0, xxlDatas.Length); + var mediumDatas = GetSizeDatas(Constants.Size.Medium); + WriteResult(isInitial: true, "Medium", mediumCount, mediumDatas.Length); + var expected = mediumDatas.Length; + Console.WriteLine($"{indent2}Changing all Medium to XXLarge"); + + var context = new Context(); + foreach (var providerData in mediumDatas) + { + // Get the old value and confirm it's as expected. We cannot have ref locals because this is an async function. + Debug.Assert(providerData.GetValue().SizeInt == (int)Constants.Size.Medium); + + // Clone the old value with updated Size; note that this cannot modify the "ref providerData.GetValue()" in-place as that will bypass PSFs. + var newValue = new TValue + { + Id = providerData.GetValue().Id, + SizeInt = (int)Constants.Size.XXLarge, // updated + ColorArgb = providerData.GetValue().ColorArgb, + Count = providerData.GetValue().Count + }; + + // Reuse the same key + if (useAsync) + await session.UpsertAsync(ref providerData.GetKey(), ref newValue, context); + else + session.Upsert(ref providerData.GetKey(), ref newValue, context, serialNo); + + RemoveIfSkippedLastBinKey(ref providerData.GetKey()); + } + ++serialNo; + + xxlDatas = GetSizeDatas(Constants.Size.XXLarge); + mediumDatas = GetSizeDatas(Constants.Size.Medium); + bool ok = xxlDatas.Length == expected && mediumDatas.Length == 0; + WriteResult(isInitial: false, "XXLarge", expected, xxlDatas.Length); + WriteResult(isInitial: false, "Medium", 0, mediumDatas.Length); + return ok; + } + + internal async static Task UpdateColorByRMW(FPSF fpsf) + where TValue : IOrders, new() + where TInput : IInput, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine(); + Console.WriteLine("Updating Colors via RMW"); + + using var session = fpsf.FasterKV.NewSession, TFunctions>(new TFunctions()); + + FasterKVProviderData[] GetColorDatas(Color color) + => useMultiGroups + ? session.QueryPSF(fpsf.ColorPsf, new ColorKey(color)).ToArray() + : session.QueryPSF(fpsf.CombinedColorPsf, new CombinedKey(color)).ToArray(); + + var purpleDatas = GetColorDatas(Color.Purple); + WriteResult(isInitial: true, "Purple", 0, purpleDatas.Length); + var blueDatas = GetColorDatas(Color.Blue); + WriteResult(isInitial: true, "Blue", blueCount, blueDatas.Length); + var expected = blueDatas.Length; + Console.WriteLine($"{indent2}Changing all Blue to Purple"); + + var context = new Context(); + var input = new TInput { IPUColorInt = Color.Purple.ToArgb() }; + foreach (var providerData in blueDatas) + { + // This will call Functions<>.InPlaceUpdater. + if (useAsync) + await session.RMWAsync(ref providerData.GetKey(), ref input, context); + else + session.RMW(ref providerData.GetKey(), ref input, context, serialNo); + + RemoveIfSkippedLastBinKey(ref providerData.GetKey()); + } + ++serialNo; + + purpleDatas = GetColorDatas(Color.Purple); + blueDatas = GetColorDatas(Color.Blue); + bool ok = purpleDatas.Length == expected && blueDatas.Length == 0; + WriteResult(isInitial: false, "Purple", expected, purpleDatas.Length); + WriteResult(isInitial: false, "Blue", 0, blueDatas.Length); + return ok; + } + + internal async static Task UpdateCountByUpsert(FPSF fpsf) + where TValue : IOrders, new() + where TInput : IInput, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine(); + Console.WriteLine("Updating Counts via Upsert"); + + using var session = fpsf.FasterKV.NewSession, TFunctions>(new TFunctions()); + + var bin7 = 7; + FasterKVProviderData[] GetCountDatas(int bin) + => useMultiGroups + ? session.QueryPSF(fpsf.CountBinPsf, new CountBinKey(bin)).ToArray() + : session.QueryPSF(fpsf.CombinedCountBinPsf, new CombinedKey(bin)).ToArray(); + + // First show we've nothing in the last bin, and get all in bin7. + var lastBinDatas = GetCountDatas(CountBinKey.LastBin); + int expectedLastBinCount = initialSkippedLastBinCount - lastBinKeys.Count(); + var ok = lastBinDatas.Length == expectedLastBinCount; + WriteResult(isInitial: true, "LastBin", expectedLastBinCount, lastBinDatas.Length); + + var bin7Datas = GetCountDatas(bin7); + ok &= bin7Datas.Length == bin7Count; + WriteResult(isInitial: true, "Bin7", bin7Count, bin7Datas.Length); + + Console.WriteLine($"{indent2}Changing all Bin7 to LastBin"); + var context = new Context(); + foreach (var providerData in bin7Datas) + { + // Get the old value and confirm it's as expected. We cannot have ref locals because this is an async function. + Debug.Assert(CountBinKey.GetBin(providerData.GetValue().Count, out int tempBin) && tempBin == bin7); + + // Clone the old value with updated Count; note that this cannot modify the "ref providerData.GetValue()" in-place as that will bypass PSFs. + var newValue = new TValue + { + Id = providerData.GetValue().Id, + SizeInt = providerData.GetValue().SizeInt, + ColorArgb = providerData.GetValue().ColorArgb, + Count = providerData.GetValue().Count + (CountBinKey.LastBin - bin7) * CountBinKey.BinSize // updated + }; + Debug.Assert(CountBinKey.GetBin(newValue.Count, out tempBin) && tempBin == CountBinKey.LastBin); + + // Reuse the same key + if (useAsync) + await session.UpsertAsync(ref providerData.GetKey(), ref newValue, context); + else + session.Upsert(ref providerData.GetKey(), ref newValue, context, serialNo); + } + ++serialNo; + + expectedLastBinCount += bin7Datas.Length; + lastBinDatas = GetCountDatas(CountBinKey.LastBin); + ok &= lastBinDatas.Length == expectedLastBinCount; + WriteResult(isInitial: false, "LastBin", expectedLastBinCount, lastBinDatas.Length); + + bin7Datas = GetCountDatas(bin7); + ok &= bin7Datas.Length == 0; + WriteResult(isInitial: false, "Bin7", 0, bin7Datas.Length); + return ok; + } + + internal async static Task Delete(FPSF fpsf) + where TValue : IOrders, new() + where TInput : IInput, new() + where TOutput : IOutput, new() + where TFunctions : IFunctions>, new() + where TSerializer : BinaryObjectSerializer, new() + { + Console.WriteLine(); + Console.WriteLine("Deleting Colors"); + + using var session = fpsf.FasterKV.NewSession, TFunctions>(new TFunctions()); + + async Task[]> GetColorDatas(Color color) + => useMultiGroups + ? useAsync + ? await session.QueryPSFAsync(fpsf.ColorPsf, new ColorKey(color)).ToArrayAsync() + : session.QueryPSF(fpsf.ColorPsf, new ColorKey(color)).ToArray() + : useAsync + ? await session.QueryPSFAsync(fpsf.CombinedColorPsf, new CombinedKey(color)).ToArrayAsync() + : session.QueryPSF(fpsf.CombinedColorPsf, new CombinedKey(color)).ToArray(); + + var redDatas = await GetColorDatas(Color.Red); + Console.WriteLine(); + Console.Write($"Deleting all Reds; initial count {redDatas.Length}"); + + var context = new Context(); + foreach (var providerData in redDatas) + { + // This will call Functions<>.InPlaceUpdater. + if (useAsync) + await session.DeleteAsync(ref providerData.GetKey(), context); + else + session.Delete(ref providerData.GetKey(), context, serialNo); + } + ++serialNo; + Console.WriteLine(); + + redDatas = await GetColorDatas(Color.Red); + var ok = redDatas.Length == 0; + Console.Write(ok ? "Passed" : "*** Failed *** "); + Console.WriteLine($": Red {redDatas.Length}"); + return ok; + } + } +} diff --git a/cs/playground/FasterPSFSample/FasterPSFSample.csproj b/cs/playground/FasterPSFSample/FasterPSFSample.csproj new file mode 100644 index 000000000..89a1694f6 --- /dev/null +++ b/cs/playground/FasterPSFSample/FasterPSFSample.csproj @@ -0,0 +1,12 @@ + + + + Exe + netcoreapp3.1 + + + + + + + diff --git a/cs/playground/FasterPSFSample/IOrders.cs b/cs/playground/FasterPSFSample/IOrders.cs new file mode 100644 index 000000000..0564ffe15 --- /dev/null +++ b/cs/playground/FasterPSFSample/IOrders.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FasterPSFSample +{ + public interface IOrders + { + int Id { get; set; } + + // Colors, strings, and enums are not blittable so we use int + int SizeInt { get; set; } + + int ColorArgb { get; set; } + + int Count { get; set; } + + (int, int, int, int) MemberTuple => (this.Id, this.SizeInt, this.ColorArgb, this.Count); + } +} diff --git a/cs/playground/FasterPSFSample/Input.cs b/cs/playground/FasterPSFSample/Input.cs new file mode 100644 index 000000000..79b628b6b --- /dev/null +++ b/cs/playground/FasterPSFSample/Input.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FasterPSFSample +{ + public interface IInput + { + TValue InitialUpdateValue { get; set; } + + int IPUColorInt { get; set; } + } + + public struct Input : IInput + { + public TValue InitialUpdateValue { get; set; } + + public int IPUColorInt { get; set; } + } +} diff --git a/cs/playground/FasterPSFSample/Key.cs b/cs/playground/FasterPSFSample/Key.cs new file mode 100644 index 000000000..43f727e3f --- /dev/null +++ b/cs/playground/FasterPSFSample/Key.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; + +namespace FasterPSFSample +{ + public struct Key : IFasterEqualityComparer + { + // Note: int instead of long because we won't use enough values to overflow and having it a + // different length than TRecordId (which is long) makes sure the PSFValue offsetting is in sync. + public int Id { get; set; } + + public Key(int id) => this.Id = id; + + public long GetHashCode64(ref Key key) => Utility.GetHashCode(key.Id); + + public bool Equals(ref Key k1, ref Key k2) => k1.Id == k2.Id; + + public (int, int, int, int) MemberTuple => FasterPSFSample.keyDict[this].MemberTuple; + + public override string ToString() => $"({this.MemberTuple})"; + } +} diff --git a/cs/playground/FasterPSFSample/LogFiles.cs b/cs/playground/FasterPSFSample/LogFiles.cs new file mode 100644 index 000000000..b9cab54d1 --- /dev/null +++ b/cs/playground/FasterPSFSample/LogFiles.cs @@ -0,0 +1,61 @@ +using FASTER.core; +using System.IO; + +namespace FasterPSFSample +{ + class LogFiles + { + private IDevice log; + private IDevice objLog; + private IDevice[] PSFDevices; + + internal LogSettings LogSettings { get; } + + internal LogSettings[] PSFLogSettings { get; } + + internal string LogDir; + + internal LogFiles(bool useObjectValue, bool useReadCache, int numPSFGroups) + { + this.LogDir = Path.Combine(Path.GetTempPath(), "FasterPSFSample"); + + // Create files for storing data. We only use one write thread to avoid disk contention. + // We set deleteOnClose to true, so logs will auto-delete on completion. + this.log = Devices.CreateLogDevice(Path.Combine(this.LogDir, "hlog.log"), deleteOnClose: true); + if (useObjectValue) + this.objLog = Devices.CreateLogDevice(Path.Combine(this.LogDir, "hlog.obj.log"), deleteOnClose: true); + + this.LogSettings = new LogSettings { LogDevice = log, ObjectLogDevice = objLog }; + if (useReadCache) + this.LogSettings.ReadCacheSettings = new ReadCacheSettings(); + + this.PSFDevices = new IDevice[numPSFGroups]; + this.PSFLogSettings = new LogSettings[numPSFGroups]; + for (var ii = 0; ii < numPSFGroups; ++ii) + { + this.PSFDevices[ii] = Devices.CreateLogDevice(Path.Combine(this.LogDir, $"psfgroup_{ii}.hlog.log"), deleteOnClose: true); + this.PSFLogSettings[ii] = new LogSettings { LogDevice = log }; + if (useReadCache) + this.PSFLogSettings[ii].ReadCacheSettings = new ReadCacheSettings(); + } + } + + internal void Close() + { + if (!(this.log is null)) + { + this.log.Close(); + this.log = null; + } + if (!(this.objLog is null)) + { + this.objLog.Close(); + this.objLog = null; + } + + foreach (var psfDevice in this.PSFDevices) + psfDevice.Close(); + this.PSFDevices = null; + } + } +} diff --git a/cs/playground/FasterPSFSample/NoSerializer.cs b/cs/playground/FasterPSFSample/NoSerializer.cs new file mode 100644 index 000000000..48af77ca5 --- /dev/null +++ b/cs/playground/FasterPSFSample/NoSerializer.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System; + +namespace FasterPSFSample +{ + public class NoSerializer : BinaryObjectSerializer + { + public override void Deserialize(out BlittableOrders obj) + => throw new NotImplementedException("NoSerializer should not be instantiated"); + + public override void Serialize(ref BlittableOrders obj) + => throw new NotImplementedException("NoSerializer should not be instantiated"); + } +} diff --git a/cs/playground/FasterPSFSample/ObjectOrders.cs b/cs/playground/FasterPSFSample/ObjectOrders.cs new file mode 100644 index 000000000..f45df2e6e --- /dev/null +++ b/cs/playground/FasterPSFSample/ObjectOrders.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; +using System; + +namespace FasterPSFSample +{ + public class ObjectOrders : IOrders + { + const int numValues = 3; + const int sizeOrd = 0; + const int colorOrd = 1; + const int countOrd = 2; + + public int Id { get; set; } + + // Colors, strings, and enums are not blittable so we use int + public int SizeInt { get => values[sizeOrd]; set => values[sizeOrd] = value; } + + public int ColorArgb { get => values[colorOrd]; set => values[colorOrd] = value; } + + public int Count { get => values[countOrd]; set => values[countOrd] = value; } + + public int[] values = new int[numValues]; + + public override string ToString() => $"{(Constants.Size)this.SizeInt}, {Constants.ColorDict[this.ColorArgb].Name}, {Count}"; + + public class Serializer : BinaryObjectSerializer + { + public override void Deserialize(out ObjectOrders obj) + { + obj = new ObjectOrders(); + obj.values = new int[numValues]; + for (var ii = 0; ii < obj.values.Length; ++ii) + obj.values[ii] = reader.ReadInt32(); + } + + public override void Serialize(ref ObjectOrders obj) + { + for (var ii = 0; ii < obj.values.Length; ++ii) + writer.Write(obj.values[ii]); + } + } + + public class Functions : IFunctions, Output, Context> + { + public ObjectOrders InitialUpdateValue { get; set; } + + #region Read + public void ConcurrentReader(ref Key key, ref Input input, ref ObjectOrders value, ref Output dst) + => dst.Value = value; + + public void SingleReader(ref Key key, ref Input input, ref ObjectOrders value, ref Output dst) + => dst.Value = value; + + public void ReadCompletionCallback(ref Key key, ref Input input, ref Output output, Context context, Status status) + { /* Output is not set by pending operations */ } + #endregion Read + + #region Upsert + public bool ConcurrentWriter(ref Key key, ref ObjectOrders src, ref ObjectOrders dst) + { + dst = src; + return true; + } + + public void SingleWriter(ref Key key, ref ObjectOrders src, ref ObjectOrders dst) + => dst = src; + + public void UpsertCompletionCallback(ref Key key, ref ObjectOrders value, Context context) + { } + #endregion Upsert + + #region RMW + public void CopyUpdater(ref Key key, ref Input input, ref ObjectOrders oldValue, ref ObjectOrders newValue) + => throw new NotImplementedException(); + + public void InitialUpdater(ref Key key, ref Input input, ref ObjectOrders value) + => value = input.InitialUpdateValue; + + public bool InPlaceUpdater(ref Key key, ref Input input, ref ObjectOrders value) + { + value.ColorArgb = input.IPUColorInt; + return true; + } + + public void RMWCompletionCallback(ref Key key, ref Input input, Context context, Status status) + { } + #endregion RMW + + public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) + { } + + public void DeleteCompletionCallback(ref Key key, Context context) + { } + } + } +} diff --git a/cs/playground/FasterPSFSample/Output.cs b/cs/playground/FasterPSFSample/Output.cs new file mode 100644 index 000000000..102e79184 --- /dev/null +++ b/cs/playground/FasterPSFSample/Output.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FasterPSFSample +{ + public interface IOutput + { + TValue Value { get; set; } + } + + public struct Output : IOutput + { + public TValue Value { get; set; } + } +} diff --git a/cs/playground/FasterPSFSample/ParseArgs.cs b/cs/playground/FasterPSFSample/ParseArgs.cs new file mode 100644 index 000000000..1cbc704c8 --- /dev/null +++ b/cs/playground/FasterPSFSample/ParseArgs.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; + +namespace FasterPSFSample +{ + public partial class FasterPSFSample + { + private static bool useObjectValue; + private static bool useMultiGroups; + private static bool useAsync; + private static bool verbose; + + const string ObjValuesArg = "--objValues"; + const string MultiGroupArg = "--multiGroup"; + const string AsyncArg = "--async"; + + static bool ParseArgs(string[] argv) + { + static bool Usage(string message = null) + { + Console.WriteLine(); + Console.WriteLine($"Usage: Run one or more Predicate Subset Functions (PSFs), specifying whether to use object or blittable (primitive) values."); + Console.WriteLine(); + Console.WriteLine($" {ObjValuesArg}: Use objects instead of blittable Value; default is {useObjectValue}"); + Console.WriteLine($" {MultiGroupArg}: Put each PSF in a separate group; default is {useMultiGroups}"); + Console.WriteLine($" {AsyncArg}: Use Async operations on FasterKV; default is {useAsync}"); + Console.WriteLine(); + if (!string.IsNullOrEmpty(message)) + { + Console.WriteLine("====== Invalid Argument(s) ======"); + Console.WriteLine(message); + Console.WriteLine(); + } + Console.WriteLine(); + return false; + } + + for (var ii = 0; ii < argv.Length; ++ii) + { + var arg = argv[ii]; + if (string.Compare(arg, ObjValuesArg, ignoreCase: true) == 0) + { + useObjectValue = true; + continue; + } + if (string.Compare(arg, MultiGroupArg, ignoreCase: true) == 0) + { + useMultiGroups = true; + continue; + } + if (string.Compare(arg, AsyncArg, ignoreCase: true) == 0) + { + useAsync = true; + continue; + } + if (string.Compare(arg, "--help", ignoreCase: true) == 0 || arg == "/?" || arg == "-?") + return Usage(); + if (string.Compare(arg, "-v", ignoreCase: true) == 0) + { + verbose = true; + continue; + } + return Usage($"Unknown argument: {arg}"); + } + return true; + } + } +} diff --git a/cs/playground/FasterPSFSample/SizeKey.cs b/cs/playground/FasterPSFSample/SizeKey.cs new file mode 100644 index 000000000..ffd2a03fd --- /dev/null +++ b/cs/playground/FasterPSFSample/SizeKey.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core; + +namespace FasterPSFSample +{ + public struct SizeKey : IFasterEqualityComparer + { + // Colors, strings, and enums are not blittable so we use int + public int SizeInt; + + public SizeKey(Constants.Size size) => this.SizeInt = (int)size; + + public override string ToString() => ((Constants.Size)this.SizeInt).ToString(); + + public long GetHashCode64(ref SizeKey key) => Utility.GetHashCode(key.SizeInt); + + public bool Equals(ref SizeKey k1, ref SizeKey k2) => k1.SizeInt == k2.SizeInt; + } +} diff --git a/cs/samples/StoreDiskReadBenchmark/Program.cs b/cs/samples/StoreDiskReadBenchmark/Program.cs index f45c670ab..3e0b52ce2 100644 --- a/cs/samples/StoreDiskReadBenchmark/Program.cs +++ b/cs/samples/StoreDiskReadBenchmark/Program.cs @@ -10,6 +10,8 @@ using System.Threading.Tasks; using FASTER.core; +#pragma warning disable CS0162 // Unreachable code detected + namespace StoreDiskReadBenchmark { public class Program diff --git a/cs/src/core/Allocator/AllocatorBase.cs b/cs/src/core/Allocator/AllocatorBase.cs index 5211a35e5..227e6543e 100644 --- a/cs/src/core/Allocator/AllocatorBase.cs +++ b/cs/src/core/Allocator/AllocatorBase.cs @@ -51,6 +51,8 @@ public unsafe abstract partial class AllocatorBase : IDisposable /// protected readonly IFasterEqualityComparer comparer; + internal KeyAccessor PsfKeyAccessor; + #region Protected size definitions /// /// Buffer size @@ -191,7 +193,7 @@ public unsafe abstract partial class AllocatorBase : IDisposable /// /// Buffer pool /// - protected SectorAlignedBufferPool bufferPool; + internal SectorAlignedBufferPool bufferPool; /// /// Read cache @@ -1486,9 +1488,14 @@ private void AsyncGetFromDiskCallback(uint errorCode, uint numBytes, object cont // We have the complete record. if (RetrievedFullRecord(record, ref ctx)) { - if (comparer.Equals(ref ctx.request_key.Get(), ref GetContextRecordKey(ref ctx))) + // If the keys are same, then I/O is complete. For PSFs, we may be querying on an address instead of a key; + // in this case, ctx.request_key is null for the primary FKV but we know the address is the one we want. + var isEqualKey = ctx.request_key is null + || (this.PsfKeyAccessor is null + ? comparer.Equals(ref ctx.request_key.Get(), ref GetContextRecordKey(ref ctx)) + : this.PsfKeyAccessor.EqualsAtRecordAddress(ref KeyPointer.CastFromKeyRef(ref ctx.request_key.Get()), (long)record)); + if (isEqualKey) { - // The keys are same, so I/O is complete // ctx.record = result.record; if (ctx.callbackQueue != null) ctx.callbackQueue.Enqueue(ctx); diff --git a/cs/src/core/Allocator/VarLenBlittableAllocator.cs b/cs/src/core/Allocator/VarLenBlittableAllocator.cs index 2fdee2118..bb66f4343 100644 --- a/cs/src/core/Allocator/VarLenBlittableAllocator.cs +++ b/cs/src/core/Allocator/VarLenBlittableAllocator.cs @@ -163,7 +163,7 @@ public override void ShallowCopy(ref Key src, ref Key dst) Unsafe.AsPointer(ref src), Unsafe.AsPointer(ref dst), KeyLength.GetLength(ref src), - KeyLength.GetLength(ref src)); + KeyLength.GetLength(ref src)); // Note: dst may not be initialized (caller ensures space is available) } public override void ShallowCopy(ref Value src, ref Value dst) @@ -172,7 +172,7 @@ public override void ShallowCopy(ref Value src, ref Value dst) Unsafe.AsPointer(ref src), Unsafe.AsPointer(ref dst), ValueLength.GetLength(ref src), - ValueLength.GetLength(ref src)); + ValueLength.GetLength(ref src)); // Note: dst may not be initialized (caller ensures space is available) } /// diff --git a/cs/src/core/ClientSession/ClientSession.cs b/cs/src/core/ClientSession/ClientSession.cs index 9fa5070a6..ec505544d 100644 --- a/cs/src/core/ClientSession/ClientSession.cs +++ b/cs/src/core/ClientSession/ClientSession.cs @@ -21,7 +21,7 @@ namespace FASTER.core /// /// /// - public sealed class ClientSession : IClientSession, IDisposable + public sealed partial class ClientSession : IClientSession, IDisposable where Functions : IFunctions { private readonly FasterKV fht; @@ -64,6 +64,8 @@ internal ClientSession( Debug.WriteLine("Warning: Session param of variableLengthStruct provided for non-varlen allocator"); } + this.CreateLazyPsfSessionWrapper(); + // Session runs on a single thread if (!supportAsync) UnsafeResumeThread(); @@ -80,10 +82,13 @@ internal ClientSession( public void Dispose() { CompletePending(true); + fht.DisposeClientSession(ID); // Session runs on a single thread if (!SupportAsync) UnsafeSuspendThread(); + + DisposeLazyPsfSessionWrapper(); } /// @@ -142,10 +147,8 @@ public Status Read(ref Key key, ref Output output, Context userContext = default /// /// ReadAsyncResult - call CompleteRead on the return value to complete the read operation [MethodImpl(MethodImplOptions.AggressiveInlining)] - public ValueTask.ReadAsyncResult> ReadAsync(ref Key key, ref Input input, Context context = default, CancellationToken token = default) - { - return fht.ReadAsync(this, ref key, ref input, context, token); - } + public ValueTask.ReadAsyncResult> ReadAsync(ref Key key, ref Input input, Context context = default, CancellationToken token = default) + => fht.ReadAsync(this, ref key, ref input, context, token); /// /// Upsert operation @@ -158,15 +161,29 @@ public ValueTask.ReadAsyncResult(); + FasterKVProviderData providerData = null; + Status status; + if (SupportAsync) UnsafeResumeThread(); try { - return fht.ContextUpsert(ref key, ref desiredValue, userContext, FasterSession, serialNo, ctx); + status = fht.ContextUpsert(ref key, ref desiredValue, userContext, this.FasterSession, serialNo, ctx, ref updateArgs); + if (status == Status.OK && this.fht.PSFManager.HasPSFs) + { + providerData = updateArgs.ChangeTracker is null + ? new FasterKVProviderData(this.fht.hlog, ref key, ref desiredValue) + : updateArgs.ChangeTracker.AfterData; + } } finally { if (SupportAsync) UnsafeSuspendThread(); } + + return providerData is null + ? status + : this.fht.PSFManager.Upsert(providerData, updateArgs.LogicalAddress, updateArgs.ChangeTracker); } /// @@ -208,15 +225,22 @@ private static async ValueTask SlowUpsertAsync(ClientSession(); + Status status; + if (SupportAsync) UnsafeResumeThread(); try { - return fht.ContextRMW(ref key, ref input, userContext, FasterSession, serialNo, ctx); + status = fht.ContextRMW(ref key, ref input, userContext, this.FasterSession, serialNo, ctx, ref updateArgs); } finally { if (SupportAsync) UnsafeSuspendThread(); } + + return (status == Status.OK || status == Status.NOTFOUND) && this.fht.PSFManager.HasPSFs + ? this.fht.PSFManager.Update(updateArgs.ChangeTracker) + : status; } /// @@ -226,18 +250,7 @@ public Status RMW(ref Key key, ref Input input, Context userContext, long serial /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status RMW(ref Key key, ref Input input) - { - if (SupportAsync) UnsafeResumeThread(); - try - { - return fht.ContextRMW(ref key, ref input, default, FasterSession, 0, ctx); - } - finally - { - if (SupportAsync) UnsafeSuspendThread(); - } - } + public Status RMW(ref Key key, ref Input input) => this.RMW(ref key, ref input, default, 0); /// /// Async RMW operation @@ -279,15 +292,22 @@ private static async ValueTask SlowRMWAsync(ClientSession(); + Status status; + if (SupportAsync) UnsafeResumeThread(); try { - return fht.ContextDelete(ref key, userContext, FasterSession, serialNo, ctx); + status = fht.ContextDelete(ref key, userContext, this.FasterSession, serialNo, ctx, ref updateArgs); } finally { if (SupportAsync) UnsafeSuspendThread(); } + + return status == Status.OK && this.fht.PSFManager.HasPSFs + ? this.fht.PSFManager.Delete(updateArgs.ChangeTracker) + : status; } /// @@ -296,18 +316,7 @@ public Status Delete(ref Key key, Context userContext, long serialNo) /// /// [MethodImpl(MethodImplOptions.AggressiveInlining)] - public Status Delete(ref Key key) - { - if (SupportAsync) UnsafeResumeThread(); - try - { - return fht.ContextDelete(ref key, default, FasterSession, 0, ctx); - } - finally - { - if (SupportAsync) UnsafeSuspendThread(); - } - } + public Status Delete(ref Key key) => this.Delete(ref key, default, 0); /// /// Async delete operation diff --git a/cs/src/core/Device/LocalStorageDevice.cs b/cs/src/core/Device/LocalStorageDevice.cs index f424dfe2c..e89c158d8 100644 --- a/cs/src/core/Device/LocalStorageDevice.cs +++ b/cs/src/core/Device/LocalStorageDevice.cs @@ -309,16 +309,25 @@ protected internal static SafeFileHandle CreateHandle(int segmentId, bool disabl if (disableFileBuffering) { - fileFlags = fileFlags | Native32.FILE_FLAG_NO_BUFFERING; + fileFlags |= Native32.FILE_FLAG_NO_BUFFERING; } if (deleteOnClose) { - fileFlags = fileFlags | Native32.FILE_FLAG_DELETE_ON_CLOSE; + fileFlags |= Native32.FILE_FLAG_DELETE_ON_CLOSE; // FILE_SHARE_DELETE allows multiple FASTER instances to share a single log directory and each can specify deleteOnClose. // This will allow the files to persist until all handles across all instances have been closed. - fileShare = fileShare | Native32.FILE_SHARE_DELETE; + fileShare |= Native32.FILE_SHARE_DELETE; + } + + var dir = Path.GetDirectoryName(fileName); + try + { + Directory.CreateDirectory(dir); + } catch (Exception ex) + { + throw new IOException($"Error creating log directory for {GetSegmentName(fileName, segmentId)}, error: {ex.Message}", ex); } var logHandle = Native32.CreateFileW( diff --git a/cs/src/core/FASTER.core.csproj b/cs/src/core/FASTER.core.csproj index c5a51a4f9..2e51ea216 100644 --- a/cs/src/core/FASTER.core.csproj +++ b/cs/src/core/FASTER.core.csproj @@ -4,6 +4,8 @@ net461;netstandard2.0;netstandard2.1 AnyCPU;x64 8 + Microsoft.FASTER + $(SolutionDir)\build_output\packages diff --git a/cs/src/core/Index/Common/Contexts.cs b/cs/src/core/Index/Common/Contexts.cs index 80c399793..91760a848 100644 --- a/cs/src/core/Index/Common/Contexts.cs +++ b/cs/src/core/Index/Common/Contexts.cs @@ -18,7 +18,10 @@ internal enum OperationType RMW, UPSERT, INSERT, - DELETE + DELETE, + PSF_READ_KEY, + PSF_READ_ADDRESS, + PSF_INSERT } internal enum OperationStatus @@ -83,6 +86,8 @@ internal struct PendingContext internal long serialNum; internal HashBucketEntry entry; internal LatchOperation heldLatch; + internal PSFReadArgs psfReadArgs; + internal PSFUpdateArgs psfUpdateArgs; public void Dispose() { diff --git a/cs/src/core/Index/FASTER/FASTER.cs b/cs/src/core/Index/FASTER/FASTER.cs index 455667d5f..bfd9e7951 100644 --- a/cs/src/core/Index/FASTER/FASTER.cs +++ b/cs/src/core/Index/FASTER/FASTER.cs @@ -201,6 +201,8 @@ public FasterKV(long size, LogSettings logSettings, systemState = default; systemState.phase = Phase.REST; systemState.version = 1; + + this.InitializePSFManager(); } /// @@ -214,14 +216,14 @@ public FasterKV(long size, LogSettings logSettings, /// public bool TakeFullCheckpoint(out Guid token) { - ISynchronizationTask backend; - if (FoldOverSnapshot) - backend = new FoldOverCheckpointTask(); - else - backend = new SnapshotCheckpointTask(); + var backend = FoldOverSnapshot ? (ISynchronizationTask)new FoldOverCheckpointTask() : new SnapshotCheckpointTask(); var result = StartStateMachine(new FullCheckpointStateMachine(backend, -1)); token = _hybridLogCheckpointToken; + + // Do not return the PSF token here. TODO: Handle failure of PSFManager.TakeFullCheckpoint + if (result && this.PSFManager.HasPSFs) + result &= this.PSFManager.TakeFullCheckpoint(); return result; } @@ -285,6 +287,10 @@ public bool TakeIndexCheckpoint(out Guid token) { var result = StartStateMachine(new IndexSnapshotStateMachine()); token = _indexCheckpointToken; + + // Do not return the PSF token here. TODO: Handle failure of PSFManager.TakeIndexCheckpoint + if (result && this.PSFManager.HasPSFs) + result &= this.PSFManager.TakeIndexCheckpoint(); return result; } @@ -317,14 +323,14 @@ public bool TakeIndexCheckpoint(out Guid token) /// Whether we could initiate the checkpoint. Use CompleteCheckpointAsync to wait completion. public bool TakeHybridLogCheckpoint(out Guid token) { - ISynchronizationTask backend; - if (FoldOverSnapshot) - backend = new FoldOverCheckpointTask(); - else - backend = new SnapshotCheckpointTask(); + var backend = FoldOverSnapshot ? (ISynchronizationTask)new FoldOverCheckpointTask() : new SnapshotCheckpointTask(); var result = StartStateMachine(new HybridLogCheckpointStateMachine(backend, -1)); token = _hybridLogCheckpointToken; + + // Do not return the PSF token here. TODO: Handle failure of PSFManager.TakeHybridLogCheckpoint + if (result && this.PSFManager.HasPSFs) + result &= this.PSFManager.TakeHybridLogCheckpoint(); return result; } @@ -378,6 +384,9 @@ public bool TakeHybridLogCheckpoint(out Guid token, CheckpointType checkpointTyp public void Recover() { InternalRecoverFromLatestCheckpoints(); + + if (this.PSFManager.HasPSFs) // TODO: Handle failure of PSFManager.Recovery + this.PSFManager.Recover(); } /// @@ -415,7 +424,7 @@ public async ValueTask CompleteCheckpointAsync(CancellationToken token = default var systemState = this.systemState; if (systemState.phase == Phase.REST || systemState.phase == Phase.PREPARE_GROW || systemState.phase == Phase.IN_PROGRESS_GROW) - return; + break; List valueTasks = new List(); @@ -430,6 +439,9 @@ public async ValueTask CompleteCheckpointAsync(CancellationToken token = default await task; } } + + if (this.PSFManager.HasPSFs) // TODO: Do in parallel and handle failure of PSFManager.CompleteCheckpointAsync + await this.PSFManager.CompleteCheckpointAsync(); } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -457,18 +469,17 @@ internal Status ContextRead(ref Key key, [MethodImpl(MethodImplOptions.AggressiveInlining)] internal Status ContextUpsert(ref Key key, ref Value value, Context context, FasterSession fasterSession, long serialNo, - FasterExecutionContext sessionCtx) + FasterExecutionContext sessionCtx, ref PSFUpdateArgs psfUpdateArgs) where FasterSession : IFasterSession { var pcontext = default(PendingContext); OperationStatus internalStatus; do - internalStatus = InternalUpsert(ref key, ref value, ref context, ref pcontext, fasterSession, sessionCtx, serialNo); + internalStatus = InternalUpsert(ref key, ref value, ref context, ref pcontext, fasterSession, sessionCtx, serialNo, ref psfUpdateArgs); while (internalStatus == OperationStatus.RETRY_NOW); Status status; - if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) { status = (Status)internalStatus; @@ -484,14 +495,14 @@ internal Status ContextUpsert(ref Key key [MethodImpl(MethodImplOptions.AggressiveInlining)] internal Status ContextRMW(ref Key key, ref Input input, Context context, FasterSession fasterSession, long serialNo, - FasterExecutionContext sessionCtx) + FasterExecutionContext sessionCtx, ref PSFUpdateArgs psfUpdateArgs) where FasterSession : IFasterSession { var pcontext = default(PendingContext); OperationStatus internalStatus; do - internalStatus = InternalRMW(ref key, ref input, ref context, ref pcontext, fasterSession, sessionCtx, serialNo); + internalStatus = InternalRMW(ref key, ref input, ref context, ref pcontext, fasterSession, sessionCtx, serialNo, ref psfUpdateArgs); while (internalStatus == OperationStatus.RETRY_NOW); Status status; @@ -514,14 +525,15 @@ internal Status ContextDelete( Context context, FasterSession fasterSession, long serialNo, - FasterExecutionContext sessionCtx) + FasterExecutionContext sessionCtx, + ref PSFUpdateArgs psfUpdateArgs) where FasterSession : IFasterSession { var pcontext = default(PendingContext); OperationStatus internalStatus; do - internalStatus = InternalDelete(ref key, ref context, ref pcontext, fasterSession, sessionCtx, serialNo); + internalStatus = InternalDelete(ref key, ref context, ref pcontext, fasterSession, sessionCtx, serialNo, ref psfUpdateArgs); while (internalStatus == OperationStatus.RETRY_NOW); Status status; diff --git a/cs/src/core/Index/FASTER/FASTERBase.cs b/cs/src/core/Index/FASTER/FASTERBase.cs index debef6709..e6a1e4946 100644 --- a/cs/src/core/Index/FASTER/FASTERBase.cs +++ b/cs/src/core/Index/FASTER/FASTERBase.cs @@ -74,6 +74,9 @@ internal static class Constants public const long kInvalidAddress = 0; public const long kTempInvalidAddress = 1; public const int kFirstValidAddress = 64; + + public const long kInvalidPsfGroupId = -1; + public const int kInvalidPsfOrdinal = 255; // 0-based ordinals; this is also the max count } [StructLayout(LayoutKind.Explicit, Size = Constants.kEntriesPerBucket * 8)] diff --git a/cs/src/core/Index/FASTER/FASTERImpl.cs b/cs/src/core/Index/FASTER/FASTERImpl.cs index f1bf58275..273053de4 100644 --- a/cs/src/core/Index/FASTER/FASTERImpl.cs +++ b/cs/src/core/Index/FASTER/FASTERImpl.cs @@ -207,6 +207,28 @@ internal OperationStatus InternalRead( #region Upsert Operation + #region PSF Utilities + private FasterKVProviderData CreateProviderData(ref Key key, long physicalAddress) + => new FasterKVProviderData(this.hlog, ref key, ref hlog.GetValue(physicalAddress)); + + private unsafe static void GetAfterRecordId(PSFChangeTracker, long> changeTracker, ref Value value) + { + // This indirection is needed because this is the primary FasterKV. + Debug.Assert(typeof(Value) == typeof(long)); + var recordId = changeTracker.AfterRecordId; + Buffer.MemoryCopy(Unsafe.AsPointer(ref recordId), Unsafe.AsPointer(ref value), sizeof(long), sizeof(long)); + } + + private void SetBeforeData(PSFChangeTracker, long> changeTracker, ref Key key, long logicalAddress, long physicalAddress, bool isIpu) + // If the value has objects, then an in-place RMW to the data in that object will also affect BeforeData, so we must get the PSFs now. // TODOperf this is in session lock + // TODOdoc: If you Read an object value and modify that fetched "ref value" directly, you will break PSFs (the before data is overwritten before we have + // a chance to see it and create the keys). An Upsert must use a separate value. + => this.PSFManager.SetBeforeData(changeTracker, CreateProviderData(ref key, physicalAddress), logicalAddress, isIpu && this.hlog.ValueHasObjects()); + + private void SetAfterData(PSFChangeTracker, long> changeTracker, ref Key key, long logicalAddress, long physicalAddress) + => this.PSFManager.SetAfterData(changeTracker, CreateProviderData(ref key, physicalAddress), logicalAddress); + #endregion PSF Utilities + /// /// Upsert operation. Replaces the value corresponding to 'key' with provided 'value', if one exists /// else inserts a new record with 'key' and 'value'. @@ -218,6 +240,7 @@ internal OperationStatus InternalRead( /// Callback functions. /// Session context /// Operation serial number + /// For PSFs, returns the inserted or updated LogicalAddress and provider data /// /// /// @@ -245,7 +268,7 @@ internal OperationStatus InternalUpsert( ref PendingContext pendingContext, FasterSession fasterSession, FasterExecutionContext sessionCtx, - long lsn) + long lsn, ref PSFUpdateArgs psfArgs) where FasterSession : IFasterSession { var status = default(OperationStatus); @@ -254,6 +277,7 @@ internal OperationStatus InternalUpsert( var logicalAddress = Constants.kInvalidAddress; var physicalAddress = default(long); var latchOperation = default(LatchOperation); + psfArgs.LogicalAddress = Constants.kInvalidAddress; var hash = comparer.GetHashCode64(ref key); var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); @@ -278,18 +302,33 @@ internal OperationStatus InternalUpsert( logicalAddress = hlog.GetInfo(physicalAddress).PreviousAddress; TraceBackForKeyMatch(ref key, logicalAddress, - hlog.ReadOnlyAddress, + this.PSFManager.HasPSFs ? hlog.HeadAddress : hlog.ReadOnlyAddress, out logicalAddress, out physicalAddress); } + + if (this.PSFManager.HasPSFs && logicalAddress >= hlog.ReadOnlyAddress && !hlog.GetInfo(physicalAddress).Tombstone) + { + // Save the PreUpdate values. + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: true); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.IPU; + } } #endregion + psfArgs.LogicalAddress = logicalAddress; + // Optimization for most common case if (sessionCtx.phase == Phase.REST && logicalAddress >= hlog.ReadOnlyAddress && !hlog.GetInfo(physicalAddress).Tombstone) { if (fasterSession.ConcurrentWriter(ref key, ref value, ref hlog.GetValue(physicalAddress))) { + if (this.PSFManager.HasPSFs) + { + SetAfterData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.IPU; + } return OperationStatus.SUCCESS; } } @@ -375,6 +414,11 @@ internal OperationStatus InternalUpsert( { if (fasterSession.ConcurrentWriter(ref key, ref value, ref hlog.GetValue(physicalAddress))) { + if (this.PSFManager.HasPSFs) + { + SetAfterData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.IPU; + } status = OperationStatus.SUCCESS; goto LatchRelease; // Release shared latch (if acquired) } @@ -392,12 +436,34 @@ internal OperationStatus InternalUpsert( var newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); RecordInfo.WriteInfo(ref hlog.GetInfo(newPhysicalAddress), sessionCtx.version, - true, false, false, + final:true, tombstone:false, invalidBit:false, latestLogicalAddress); hlog.ShallowCopy(ref key, ref hlog.GetKey(newPhysicalAddress)); fasterSession.SingleWriter(ref key, ref value, ref hlog.GetValue(newPhysicalAddress)); + if (this.PSFManager.HasPSFs) + { + psfArgs.LogicalAddress = newLogicalAddress; + + if (logicalAddress < hlog.HeadAddress || hlog.GetInfo(physicalAddress).Tombstone) + { + // Old logicalAddress is invalid (LA < BeginAddress, so the record does not exist), on disk (LA < HeadAddress; this + // would require an IO to get it, so instead we defer to the liveness check in PsfInternalReadAddress), or was deleted, + // so this is an insert. This goes through the fast Insert path which does not create a changeTracker. + } + else + { + // The old record was valid but not in the mutable range (that's handled above), but it's above HeadAddress, so we can + // get the old value and make this an RCU. + Debug.Assert(logicalAddress < hlog.ReadOnlyAddress); + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: false); + SetAfterData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.RCU; + } + } + var updatedEntry = default(HashBucketEntry); updatedEntry.Tag = tag; updatedEntry.Address = newLogicalAddress & Constants.kAddressMask; @@ -434,6 +500,9 @@ internal OperationStatus InternalUpsert( pendingContext.logicalAddress = logicalAddress; pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; + + psfArgs.ChangeTracker = null; + pendingContext.psfUpdateArgs = psfArgs; } #endregion @@ -473,6 +542,7 @@ internal OperationStatus InternalUpsert( /// Callback functions. /// Session context /// Operation serial number + /// For PSFs, returns the updated LogicalAddress and provider data /// /// /// @@ -504,7 +574,7 @@ internal OperationStatus InternalRMW( ref PendingContext pendingContext, FasterSession fasterSession, FasterExecutionContext sessionCtx, - long lsn) + long lsn, ref PSFUpdateArgs psfArgs) where FasterSession : IFasterSession { var recordSize = default(int); @@ -544,6 +614,14 @@ internal OperationStatus InternalRMW( out logicalAddress, out physicalAddress); } + + if (this.PSFManager.HasPSFs && logicalAddress >= hlog.ReadOnlyAddress && !hlog.GetInfo(physicalAddress).Tombstone) + { + // Get the PreUpdate values (or the secondary FKV position in the IPUCache). + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: true); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.IPU; + } } #endregion @@ -552,6 +630,8 @@ internal OperationStatus InternalRMW( { if (fasterSession.InPlaceUpdater(ref key, ref input, ref hlog.GetValue(physicalAddress))) { + if (!(psfArgs.ChangeTracker is null)) + SetAfterData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); return OperationStatus.SUCCESS; } } @@ -647,6 +727,8 @@ internal OperationStatus InternalRMW( if (fasterSession.InPlaceUpdater(ref key, ref input, ref hlog.GetValue(physicalAddress))) { + if (!(psfArgs.ChangeTracker is null)) + SetAfterData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress); status = OperationStatus.SUCCESS; goto LatchRelease; // Release shared latch (if acquired) } @@ -736,10 +818,32 @@ ref hlog.GetValue(physicalAddress), { // ah, old record slipped onto disk hlog.GetInfo(newPhysicalAddress).Invalid = true; + psfArgs.ChangeTracker = null; status = OperationStatus.RETRY_NOW; goto LatchRelease; } + if (this.PSFManager.HasPSFs) + { + psfArgs.LogicalAddress = newLogicalAddress; + var isInsert = logicalAddress < hlog.BeginAddress || hlog.GetInfo(physicalAddress).Tombstone; + if (isInsert) + { + // Old logicalAddress is invalid or deleted, so this is an Insert only. + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress, isIpu: false); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Insert; + } + else + { + // The old record was valid but not in mutable range (that's handled above), so this is an RCU + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: false); + SetAfterData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.RCU; + } + } + var updatedEntry = default(HashBucketEntry); updatedEntry.Tag = tag; updatedEntry.Address = newLogicalAddress & Constants.kAddressMask; @@ -777,6 +881,9 @@ ref hlog.GetValue(physicalAddress), pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; pendingContext.heldLatch = heldOperation; + + psfArgs.ChangeTracker = null; + pendingContext.psfUpdateArgs = psfArgs; } #endregion @@ -814,6 +921,7 @@ ref hlog.GetValue(physicalAddress), /// Callback functions. /// Session context /// Operation serial number + /// For PSFs, returns the updated LogicalAddress and provider data /// /// /// @@ -841,7 +949,7 @@ internal OperationStatus InternalDelete( ref PendingContext pendingContext, FasterSession fasterSession, FasterExecutionContext sessionCtx, - long lsn) + long lsn, ref PSFUpdateArgs psfArgs) where FasterSession : IFasterSession { var status = default(OperationStatus); @@ -980,6 +1088,14 @@ internal OperationStatus InternalDelete( // Apply tombstone bit to the record hlog.GetInfo(physicalAddress).Tombstone = true; + if (this.PSFManager.HasPSFs) + { + // Get the PreUpdate values (or the secondary FKV position in the IPUCache). + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: false); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Delete; + } + if (WriteDefaultOnDelete) { // Write default value @@ -999,6 +1115,14 @@ internal OperationStatus InternalDelete( { hlog.GetInfo(physicalAddress).Tombstone = true; + if (this.PSFManager.HasPSFs) + { + // Get the PreUpdate values (or the secondary FKV position in the IPUCache). + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, logicalAddress, physicalAddress, isIpu: false); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Delete; + } + if (WriteDefaultOnDelete) { // Write default value @@ -1042,6 +1166,14 @@ internal OperationStatus InternalDelete( if (foundEntry.word == entry.word) { + if (this.PSFManager.HasPSFs) + { + // Get the PreUpdate values (or the secondary FKV position in the IPUCache). + psfArgs.ChangeTracker = this.PSFManager.CreateChangeTracker(); + SetBeforeData(psfArgs.ChangeTracker, ref key, newLogicalAddress, newPhysicalAddress, isIpu: false); + psfArgs.ChangeTracker.UpdateOp = UpdateOperation.Delete; + } + status = OperationStatus.SUCCESS; goto LatchRelease; } @@ -1064,6 +1196,9 @@ internal OperationStatus InternalDelete( pendingContext.logicalAddress = logicalAddress; pendingContext.version = sessionCtx.version; pendingContext.serialNum = lsn; + + psfArgs.ChangeTracker = null; + pendingContext.psfUpdateArgs = psfArgs; } #endregion @@ -1190,13 +1325,44 @@ internal OperationStatus InternalContinuePendingRead; + functions.VisitSecondaryRead(ref hlog.GetContextRecordKey(ref request), + ref hlog.GetContextRecordValue(ref request), + ref pendingContext.input, ref pendingContext.output, + tombstone: false, // checked above + isConcurrent: false); + } + else if (pendingContext.type == OperationType.PSF_READ_ADDRESS) + { + var functions = fasterSession as IPSFFunctions; + functions.VisitSecondaryRead(ref hlog.GetContextRecordKey(ref request), + ref hlog.GetContextRecordValue(ref request), + ref pendingContext.input, ref pendingContext.output, + tombstone: false, // checked above + isConcurrent: false); + } + + if (tombstone) + return OperationStatus.NOTFOUND; + + if (CopyReadsToTail || UseReadCache && !this.ImplmentsPSFs) // TODOdcr: Support ReadCache and CopyReadsToTail for PSFs { InternalContinuePendingReadCopyToTail(ctx, request, ref pendingContext, fasterSession, currentCtx); } @@ -1450,7 +1616,8 @@ ref hlog.GetContextRecordValue(ref request), Retry: OperationStatus internalStatus; do - internalStatus = InternalRMW(ref pendingContext.key.Get(), ref pendingContext.input, ref pendingContext.userContext, ref pendingContext, fasterSession, opCtx, pendingContext.serialNum); + internalStatus = InternalRMW(ref pendingContext.key.Get(), ref pendingContext.input, ref pendingContext.userContext, ref pendingContext, + fasterSession, opCtx, pendingContext.serialNum, ref pendingContext.psfUpdateArgs); while (internalStatus == OperationStatus.RETRY_NOW); return internalStatus; } @@ -1510,24 +1677,48 @@ internal Status HandleOperationStatus( ref pendingContext.userContext, ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum); break; + case OperationType.PSF_READ_KEY: + internalStatus = PsfInternalReadKey(ref pendingContext.key.Get(), + ref pendingContext.input, + ref pendingContext.output, + ref pendingContext.userContext, + ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum); + break; + case OperationType.PSF_READ_ADDRESS: + internalStatus = PsfInternalReadAddress(ref pendingContext.input, + ref pendingContext.output, + ref pendingContext.userContext, + ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum); + break; case OperationType.UPSERT: internalStatus = InternalUpsert(ref pendingContext.key.Get(), ref pendingContext.value.Get(), ref pendingContext.userContext, + ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum, + ref pendingContext.psfUpdateArgs); + break; + case OperationType.PSF_INSERT: + internalStatus = PsfInternalInsert(ref pendingContext.key.Get(), + ref pendingContext.value.Get(), + ref pendingContext.input, + ref pendingContext.userContext, ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum); break; case OperationType.DELETE: internalStatus = InternalDelete(ref pendingContext.key.Get(), ref pendingContext.userContext, - ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum); + ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum, + ref pendingContext.psfUpdateArgs); break; case OperationType.RMW: internalStatus = InternalRMW(ref pendingContext.key.Get(), ref pendingContext.input, ref pendingContext.userContext, - ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum); + ref pendingContext, fasterSession, currentCtx, pendingContext.serialNum, + ref pendingContext.psfUpdateArgs); break; } + Debug.Assert(internalStatus != OperationStatus.CPR_SHIFT_DETECTED); } while (internalStatus == OperationStatus.RETRY_NOW); status = internalStatus; @@ -1684,6 +1875,7 @@ private bool TraceBackForKeyMatch( out long foundLogicalAddress, out long foundPhysicalAddress) { + Debug.Assert(!this.ImplmentsPSFs); foundLogicalAddress = fromLogicalAddress; while (foundLogicalAddress >= minOffset) { @@ -1912,8 +2104,9 @@ private bool ReadFromCache(ref Key key, ref long logicalAddress, ref long physic { if ((logicalAddress & ~Constants.kReadCacheBitMask) >= readcache.SafeReadOnlyAddress) { - return true; + return true; } + Debug.Assert((logicalAddress & ~Constants.kReadCacheBitMask) >= readcache.SafeHeadAddress); // TODO: copy to tail of read cache // and return new cache entry diff --git a/cs/src/core/Index/FASTER/FASTERLegacy.cs b/cs/src/core/Index/FASTER/FASTERLegacy.cs index 5d1fff620..44734ac22 100644 --- a/cs/src/core/Index/FASTER/FASTERLegacy.cs +++ b/cs/src/core/Index/FASTER/FASTERLegacy.cs @@ -158,7 +158,8 @@ public Status Read(ref Key key, ref Input input, ref Output output, Context cont [Obsolete("Use NewSession() and invoke Upsert() on the session.")] public Status Upsert(ref Key key, ref Value value, Context context, long serialNo) { - return _fasterKV.ContextUpsert(ref key, ref value, context, FasterSession, serialNo, _threadCtx.Value); + PSFUpdateArgs psfUpdateArgs = default; + return _fasterKV.ContextUpsert(ref key, ref value, context, this.FasterSession, serialNo, _threadCtx.Value, ref psfUpdateArgs); } /// @@ -172,7 +173,8 @@ public Status Upsert(ref Key key, ref Value value, Context context, long serialN [Obsolete("Use NewSession() and invoke RMW() on the session.")] public Status RMW(ref Key key, ref Input input, Context context, long serialNo) { - return _fasterKV.ContextRMW(ref key, ref input, context, FasterSession, serialNo, _threadCtx.Value); + PSFUpdateArgs psfUpdateArgs = default; + return _fasterKV.ContextRMW(ref key, ref input, context, this.FasterSession, serialNo, _threadCtx.Value, ref psfUpdateArgs); } /// @@ -188,7 +190,8 @@ public Status RMW(ref Key key, ref Input input, Context context, long serialNo) [Obsolete("Use NewSession() and invoke Delete() on the session.")] public Status Delete(ref Key key, Context context, long serialNo) { - return _fasterKV.ContextDelete(ref key, context, FasterSession, serialNo, _threadCtx.Value); + PSFUpdateArgs psfUpdateArgs = default; + return _fasterKV.ContextDelete(ref key, context, this.FasterSession, serialNo, _threadCtx.Value, ref psfUpdateArgs); } /// diff --git a/cs/src/core/Index/FASTER/FASTERThread.cs b/cs/src/core/Index/FASTER/FASTERThread.cs index 7c39283bd..57415f0bc 100644 --- a/cs/src/core/Index/FASTER/FASTERThread.cs +++ b/cs/src/core/Index/FASTER/FASTERThread.cs @@ -214,23 +214,33 @@ internal void InternalCompleteRetryRequest(this.hlog, ref key, ref value), + pendingContext.psfUpdateArgs.LogicalAddress, + pendingContext.psfUpdateArgs.ChangeTracker); + break; + case OperationType.PSF_INSERT: + var functions = fasterSession as IPSFFunctions; + var updateOp = pendingContext.psfUpdateArgs.ChangeTracker.UpdateOp; + if (functions.IsDelete(ref pendingContext.input) && (updateOp == UpdateOperation.IPU || updateOp == UpdateOperation.RCU)) + { + // RCU Insert of a tombstoned old record is followed by Insert of the new record. + if (pendingContext.psfUpdateArgs.ChangeTracker.FindGroup(functions.GroupId(ref pendingContext.input), out var ordinal)) + { + ref GroupCompositeKeyPair groupKeysPair = ref pendingContext.psfUpdateArgs.ChangeTracker.GetGroupRef(ordinal); + GetAfterRecordId(pendingContext.psfUpdateArgs.ChangeTracker, ref value); + var pcontext = default(PendingContext); + PsfRcuInsert(groupKeysPair.After, ref value, ref pendingContext.input, ref pendingContext.userContext, + ref pcontext, fasterSession, currentCtx, pendingContext.serialNum + 1); + } + } + break; + default: + break; + } + } + // If done, callback user code. if (status == Status.OK || status == Status.NOTFOUND) { @@ -270,7 +311,6 @@ ref pendingContext.value.Get(), default: throw new FasterException("Operation type not allowed for retry"); } - } } #endregion @@ -331,20 +371,21 @@ internal void InternalCompletePendingRequest {internalStatus}"); } + ref Key key = ref pendingContext.key.Get(); + if (pendingContext.heldLatch == LatchOperation.Shared) ReleaseSharedLatch(key); diff --git a/cs/src/core/Index/Interfaces/IFasterKV.cs b/cs/src/core/Index/Interfaces/IFasterKV.cs index 64b376ec2..1c70648f4 100644 --- a/cs/src/core/Index/Interfaces/IFasterKV.cs +++ b/cs/src/core/Index/Interfaces/IFasterKV.cs @@ -251,6 +251,106 @@ ClientSession ResumeSession + /// Register a with a simple definition. + /// + /// + /// static TPSFKey? sizePsfFunc(ref TKVKey key, ref TKVValue value) => new TPSFKey(value.size); + /// var sizePsfDef = new FasterKVPSFDefinition{TKVKey, TKVValue, TPSFKey}("sizePSF", sizePsfFunc); + /// var sizePsf = fht.RegisterPSF(psfRegistrationSettings, sizePsfDef); + /// + /// The type of the key value returned from the + /// Registration settings for the secondary FasterKV instances, etc. + /// A FasterKV-specific form of a PSF definition + /// A FasterKV-specific PSF implementation whose TRecordId is long( + IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, + FasterKVPSFDefinition def) + where TPSFKey : struct; + + /// + /// Register a with a simple definition. + /// + /// + /// static TPSFKey? sizePsfFunc(ref TKVKey key, ref TKVValue value) => new TPSFKey(value.size); + /// var sizePsfDef = new FasterKVPSFDefinition{TKVKey, TKVValue, TPSFKey}("sizePSF", sizePsfFunc); + /// var sizePsf = fht.RegisterPSF(psfRegistrationSettings, new [] { sizePsfDef }); + /// + /// The type of the key value returned from the + /// Registration settings for the secondary FasterKV instances, etc. + /// An array of FasterKV-specific forms of PSF definitions + /// A FasterKV-specific PSF implementation whose TRecordId is long( + IPSF[] RegisterPSF(PSFRegistrationSettings registrationSettings, + params FasterKVPSFDefinition[] defs) + where TPSFKey : struct; + + /// + /// Register a with a simple definition. + /// + /// + /// var sizePsf = fht.RegisterPSF(psfRegistrationSettings, "sizePsf", (k, v) => new TPSFKey(v.size)); + /// + /// The type of the key value returned from the + /// Registration settings for the secondary FasterKV instances, etc. + /// The name of the PSF; must be unique across all PSFGroups in this FasterKV instance + /// A Func implementing the PSF, it will be wrapped in a delegate + /// A FasterKV-specific implementation whose TRecordId is long + IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, + string psfName, Func psfFunc) + where TPSFKey : struct; + + /// + /// Register multiple with no registration settings. + /// + /// + /// var sizePsf = fht.RegisterPSF(psfRegistrationSettings, + /// ("sizePsf", (k, v) => new TPSFKey(v.size)), + /// ("colorPsf", (k, v) => new TPSFKey(v.color))); + /// + /// The type of the key value returned from the + /// Registration settings for the secondary FasterKV instances, etc. + /// One or more tuples containing a PSF name and implementing Func; the name must be + /// unique across all PSFGroups in this FasterKV instance, and the Func will be wrapped in a delegate + /// "params" won't allow the optional fromAddress and keyComparer, so an overload is provided + /// to specify those + IPSF[] RegisterPSF(PSFRegistrationSettings registrationSettings, + params (string, Func)[] psfFuncs) + where TPSFKey : struct; + + /// + /// Returns the names of registered s for use in recovery. + /// TODO: Supplement or replace this with an app version string. + /// + /// An array of string arrays; each outer array corresponds to a + /// + string[][] GetRegisteredPSFNames(); + + #endregion PSF Registration + + #region PSF Logs + // TODO: better interface to PSF logs + + /// + /// Flush PSF logs until current tail (records are still retained in memory) + /// + /// Synchronous wait for operation to complete + void FlushPSFLogs(bool wait); + + /// + /// Flush PSF logs and evict all records from memory + /// + /// Synchronous wait for operation to complete + /// When wait is false, this tells whether the full eviction was successfully registered with FASTER + public void FlushAndEvictPSFLogs(bool wait); + + /// + /// Delete PSF logs entirely from memory. Cannot allocate on the log + /// after this point. This is a synchronous operation. + /// + public void DisposePSFLogsFromMemory(); + + #endregion PSF Logs + #region Growth and Recovery /// diff --git a/cs/src/core/Index/PSF/CompositeKey.cs b/cs/src/core/Index/PSF/CompositeKey.cs new file mode 100644 index 000000000..fe0b31aec --- /dev/null +++ b/cs/src/core/Index/PSF/CompositeKey.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + /// + /// Wraps the set of TPSFKeys for a record in the secondary FasterKV instance. + /// + /// + public unsafe struct CompositeKey + { + // This class is essentially a "reinterpret_cast*>" implementation; there are no data members. + + /// + /// Get a reference to the key for the PSF identified by psfOrdinal. + /// + /// The ordinal of the PSF in its parent PSFGroup + /// Size of the KeyPointer{TPSFKey} struct + /// A reference to the key for the PSF identified by psfOrdinal. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ref KeyPointer GetKeyPointerRef(int psfOrdinal, int keyPointerSize) + => ref Unsafe.AsRef>((byte*)Unsafe.AsPointer(ref this) + keyPointerSize * psfOrdinal); + + /// + /// Get a reference to the key for the PSF identified by psfOrdinal. + /// + /// The ordinal of the PSF in its parent PSFGroup + /// Size of the struct + /// A reference to the key for the PSF identified by psfOrdinal. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ref TPSFKey GetKeyRef(int psfOrdinal, int keyPointerSize) + => ref GetKeyPointerRef(psfOrdinal, keyPointerSize).Key; + + /// + /// Returns a reference to the CompositeKey from a reference to the first + /// + /// A reference to the first , typed as TPSFKey + /// Used when converting the CompositeKey to/from the TPSFKey type for secondary FKV operations + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static ref CompositeKey CastFromFirstKeyPointerRefAsKeyRef(ref TPSFKey firstKeyPointerRef) + => ref Unsafe.AsRef>((byte*)Unsafe.AsPointer(ref firstKeyPointerRef)); + + /// + /// Converts this CompositeKey reference to a reference to the first , typed as TPSFKey. + /// + /// Used when converting the CompositeKey to/from the TPSFKey type for secondary FKV operations + /// A reference to the first , typed as TPSFKey + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ref TPSFKey CastToFirstKeyPointerRefAsKeyRef() + => ref Unsafe.AsRef((byte*)Unsafe.AsPointer(ref this)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void ClearUpdateFlags(int psfCount, int keyPointerSize) + { + for (var ii = 0; ii < psfCount; ++ii) + this.GetKeyPointerRef(0, keyPointerSize).ClearUpdateFlags(); + } + + internal class VarLenLength : IVariableLengthStruct + { + private readonly int size; + + internal VarLenLength(int keyPointerSize, int psfCount) => this.size = keyPointerSize * psfCount; + + public int GetInitialLength() => this.size; + + public int GetLength(ref TPSFKey _) => this.size; + } + + /// + /// This is the unused key comparer passed to the secondary FasterKV + /// + internal class UnusedKeyComparer : IFasterEqualityComparer + { + public long GetHashCode64(ref TPSFKey cKey) + => throw new PSFInternalErrorException("Must use KeyAccessor instead (psfOrdinal is required)"); + + public bool Equals(ref TPSFKey cKey1, ref TPSFKey cKey2) + => throw new PSFInternalErrorException("Must use KeyAccessor instead (psfOrdinal is required)"); + } + } +} diff --git a/cs/src/core/Index/PSF/DeadRecords.cs b/cs/src/core/Index/PSF/DeadRecords.cs new file mode 100644 index 000000000..b8deddba5 --- /dev/null +++ b/cs/src/core/Index/PSF/DeadRecords.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace FASTER.core.Index.PSF +{ + internal struct DeadRecords + where TRecordId : struct + { + private HashSet deadRecs; + + internal void Add(TRecordId recordId) + { + this.deadRecs ??= new HashSet(); + this.deadRecs.Add(recordId); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool IsDead(TRecordId recordId, bool thisInstanceIsDead) + { + if (thisInstanceIsDead) + { + this.Add(recordId); + return true; + } + return ContainsAndUpdate(recordId, thisInstanceIsDead); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal bool ContainsAndUpdate(TRecordId recordId, bool thisInstanceIsDead) + { + if (this.deadRecs is null || !this.deadRecs.Contains(recordId)) + return false; + if (!thisInstanceIsDead) // A live record will not be encountered again so remove it + this.Remove(recordId); + return true; + } + + internal void Remove(TRecordId recordId) + { + if (!(this.deadRecs is null)) + this.deadRecs.Remove(recordId); + } + } +} diff --git a/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs b/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs new file mode 100644 index 000000000..59acda16f --- /dev/null +++ b/cs/src/core/Index/PSF/FasterKVPSFDefinition.cs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; + +namespace FASTER.core +{ + /// + /// The definition of a single PSF (Predicate Subset Function) + /// + /// The type of the key in the primary FasterKV instance + /// The type of the value in the primary FasterKV instance + /// The type of the key returned by the Predicate and store in the secondary + /// (PSF-implementing) FasterKV instances + public class FasterKVPSFDefinition : IPSFDefinition, TPSFKey> + where TPSFKey : struct + { + /// + /// The definition of the delegate used to obtain a new key matching the Value for this PSF definition. + /// + /// The key sent to FasterKV on Upsert or RMW + /// The value sent to FasterKV on Upsert or RMW + /// This must be a delegate instead of a lambda to allow ref parameters + /// Null if the value does not match the predicate, else a key for the value in the PSF hash table + public delegate TPSFKey? PredicateFunc(ref TKVKey kvKey, ref TKVValue kvValue); + + /// + /// The predicate function that will be called by FasterKV on Upsert or RMW. + /// + public PredicateFunc Predicate; + + /// + /// Executes the Predicate + /// + /// The record obtained from the primary FasterKV instance + /// + /// Null if the value does not match the predicate, else a key for the value in the PSF hash table + public TPSFKey? Execute(FasterKVProviderData record) + => Predicate(ref record.GetKey(), ref record.GetValue()); + + /// + /// The Name of the PSF, assigned by the caller. Must be unique among all PSFs. + /// + public string Name { get; } + + /// + /// Instantiates the instance with the name and predicate delegate + /// + /// + /// + public FasterKVPSFDefinition(string name, PredicateFunc predicate) + { + this.Name = name; + this.Predicate = predicate; + } + + /// + /// Instantiates the instance with the name and predicate Func{}, which we wrap in a delegate. + /// This allows a streamlined API call. + /// + /// + /// + public FasterKVPSFDefinition(string name, Func predicate) + { + TPSFKey? wrappedPredicate(ref TKVKey key, ref TKVValue value) => predicate(key, value); + + this.Name = name; + this.Predicate = wrappedPredicate; + } + } +} + \ No newline at end of file diff --git a/cs/src/core/Index/PSF/FasterKVProviderData.cs b/cs/src/core/Index/PSF/FasterKVProviderData.cs new file mode 100644 index 000000000..35ccf8283 --- /dev/null +++ b/cs/src/core/Index/PSF/FasterKVProviderData.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FASTER.core +{ + /// + /// The wrapper around the provider data stored in the primary faster instance. + /// + /// The type of the key in the primary FasterKV instance + /// The type of the value in the primary FasterKV instance + /// Having this enables separation between the LogicalAddress stored in the PSF-implementing + /// FasterKV instances, and the actual and + /// types. + public class FasterKVProviderData : IDisposable + { + // C# doesn't allow ref fields and even if it did, if the client held the FasterKVProviderData + // past the ref lifetime, bad things would happen when accessing the ref key/value. + internal IHeapContainer keyContainer; + internal IHeapContainer valueContainer; + + internal FasterKVProviderData(AllocatorBase allocator, ref TKVKey key, ref TKVValue value) + { + this.keyContainer = allocator.GetKeyContainer(ref key); + this.valueContainer = allocator.GetValueContainer(ref value); + } + + public unsafe ref TKVKey GetKey() => ref this.keyContainer.Get(); + + public unsafe ref TKVValue GetValue() => ref this.valueContainer.Get(); + + /// + public void Dispose() + { + this.keyContainer.Dispose(); + this.valueContainer.Dispose(); + } + + public override string ToString() + { + return $"Key = {(this.keyContainer is null ? "-0-" : this.keyContainer.Get().ToString())};" + + $" Value = {(this.valueContainer is null ? "-0-" : this.valueContainer.Get().ToString())}"; + } + } +} diff --git a/cs/src/core/Index/PSF/FasterPSFContextOperations.cs b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs new file mode 100644 index 000000000..14fdc7fe3 --- /dev/null +++ b/cs/src/core/Index/PSF/FasterPSFContextOperations.cs @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; + +namespace FASTER.core +{ + public partial class FasterKV : FasterBase, IFasterKV + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfReadKey(ref Key key, ref Input input, ref Output output, ref Context context, + FasterSession fasterSession, long serialNo, FasterExecutionContext sessionCtx) + where FasterSession : IFasterSession + { + var pcontext = default(PendingContext); + var internalStatus = this.PsfInternalReadKey(ref key, ref input, ref output, ref context, ref pcontext, fasterSession, sessionCtx, serialNo); + var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, fasterSession, internalStatus); + + sessionCtx.serialNum = serialNo; + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ValueTask> ContextPsfReadKeyAsync( + ClientSession clientSession, + ref Key key, ref Input input, ref Output output, ref Context context, long serialNo, + FasterExecutionContext sessionCtx, PSFQuerySettings querySettings) + where Functions : IFunctions + { + return ContextPsfReadAsync(clientSession, isKey: true, ref key, ref input, ref output, ref context, serialNo, sessionCtx, querySettings); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfReadAddress(ref Input input, ref Output output, ref Context context, + FasterSession fasterSession, long serialNo, FasterExecutionContext sessionCtx) + where FasterSession : IFasterSession + { + var pcontext = default(PendingContext); + var internalStatus = this.PsfInternalReadAddress(ref input, ref output, ref context, ref pcontext, fasterSession, sessionCtx, serialNo); + var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, fasterSession, internalStatus); + + sessionCtx.serialNum = serialNo; + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ValueTask> ContextPsfReadAddressAsync( + ClientSession clientSession, + ref Input input, ref Output output, ref Context context, long serialNo, + FasterExecutionContext sessionCtx, PSFQuerySettings querySettings) + where Functions : IFunctions + { + var key = default(Key); + return ContextPsfReadAsync(clientSession, isKey: false, ref key, ref input, ref output, ref context, serialNo, sessionCtx, querySettings); + } + + internal ValueTask> ContextPsfReadAsync( + ClientSession clientSession, bool isKey, + ref Key key, ref Input input, ref Output output, ref Context context, long serialNo, + FasterExecutionContext sessionCtx, PSFQuerySettings querySettings) + where Functions : IFunctions + { + var pcontext = default(PendingContext); + var nextSerialNum = clientSession.ctx.serialNum + 1; + + if (clientSession.SupportAsync) clientSession.UnsafeResumeThread(); + try + { + TryReadAgain: + var internalStatus = isKey + ? this.PsfInternalReadKey(ref key, ref input, ref output, ref context, ref pcontext, clientSession.FasterSession, sessionCtx, serialNo) + : this.PsfInternalReadAddress(ref input, ref output, ref context, ref pcontext, clientSession.FasterSession, sessionCtx, serialNo); + if (internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND) + { + return new ValueTask>(new ReadAsyncResult((Status)internalStatus, output)); + } + + if (internalStatus == OperationStatus.CPR_SHIFT_DETECTED) + { + SynchronizeEpoch(clientSession.ctx, clientSession.ctx, ref pcontext, clientSession.FasterSession); + goto TryReadAgain; + } + } + finally + { + clientSession.ctx.serialNum = nextSerialNum; + if (clientSession.SupportAsync) clientSession.UnsafeSuspendThread(); + } + + try + { + return SlowReadAsync(this, clientSession, pcontext, querySettings.CancellationToken); + } + catch (OperationCanceledException) when (!querySettings.ThrowOnCancellation) + { + return default; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfInsert(ref Key key, ref Value value, + ref Input input, ref Context context, + FasterSession fasterSession, long serialNo, + FasterExecutionContext sessionCtx) + where FasterSession : IFasterSession + { + var pcontext = default(PendingContext); + var internalStatus = this.PsfInternalInsert(ref key, ref value, ref input, ref context, + ref pcontext, fasterSession, sessionCtx, serialNo); + var status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, fasterSession, internalStatus); + + sessionCtx.serialNum = serialNo; + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfUpdate(ref GroupCompositeKeyPair groupKeysPair, ref Value value, + ref Input input, ref Context context, + FasterSession fasterSession, long serialNo, + FasterExecutionContext sessionCtx, + PSFChangeTracker changeTracker) + where FasterSession : IFasterSession + { + var pcontext = default(PendingContext); + var groupKeys = groupKeysPair.Before; + + var functions = GetFunctions(ref context); + functions.SetDelete(ref input, true); + + var internalStatus = this.PsfInternalInsert(ref groupKeys.CastToKeyRef(), ref value, ref input, ref context, + ref pcontext, fasterSession, sessionCtx, serialNo); + Status status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, fasterSession, internalStatus); + + sessionCtx.serialNum = serialNo; + + if (status == Status.OK) + { + value = changeTracker.AfterRecordId; + return PsfRcuInsert(groupKeysPair.After, ref value, ref input, ref context, ref pcontext, fasterSession, sessionCtx, serialNo + 1); + } + return status; + } + + private Status PsfRcuInsert(GroupCompositeKey groupKeys, ref Value value, ref Input input, + ref Context context, ref PendingContext pcontext, FasterSession fasterSession, + FasterExecutionContext sessionCtx, long serialNo) + where FasterSession : IFasterSession + { + var functions = GetFunctions(ref context); + functions.SetDelete(ref input, false); + var internalStatus = this.PsfInternalInsert(ref groupKeys.CastToKeyRef(), ref value, ref input, ref context, + ref pcontext, fasterSession, sessionCtx, serialNo); + return internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, fasterSession, internalStatus); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal Status ContextPsfDelete(ref Key key, ref Value value, ref Input input, + ref Context context, FasterSession fasterSession, long serialNo, + FasterExecutionContext sessionCtx, + PSFChangeTracker changeTracker) + where FasterSession : IFasterSession + { + var pcontext = default(PendingContext); + + var functions = GetFunctions(ref context); + functions.SetDelete(ref input, true); + + var internalStatus = this.PsfInternalInsert(ref key, ref value, ref input, ref context, ref pcontext, fasterSession, sessionCtx, serialNo); + Status status = internalStatus == OperationStatus.SUCCESS || internalStatus == OperationStatus.NOTFOUND + ? (Status)internalStatus + : HandleOperationStatus(sessionCtx, sessionCtx, pcontext, fasterSession, internalStatus); + + sessionCtx.serialNum = serialNo; + return status; + } + } +} diff --git a/cs/src/core/Index/PSF/FasterPSFImpl.cs b/cs/src/core/Index/PSF/FasterPSFImpl.cs new file mode 100644 index 000000000..e917b2bb7 --- /dev/null +++ b/cs/src/core/Index/PSF/FasterPSFImpl.cs @@ -0,0 +1,521 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +//#define PSF_TRACE + +using System; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Threading; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FASTER.core +{ + // PSF-related internal function implementations for FasterKV; these correspond to the similarly-named + // functions in FasterImpl.cs. + public unsafe partial class FasterKV : FasterBase, IFasterKV + { + internal KeyAccessor PsfKeyAccessor => this.hlog.PsfKeyAccessor; + + internal bool ImplmentsPSFs => !(this.PsfKeyAccessor is null); + + bool ScanQueryChain(ref long logicalAddress, ref KeyPointer queryKeyPointer, ref int latestRecordVersion) + { + long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + var recordAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + if (latestRecordVersion == -1) + latestRecordVersion = hlog.GetInfo(recordAddress).Version; + + while (true) + { + if (this.PsfKeyAccessor.EqualsAtKeyAddress(ref queryKeyPointer, physicalAddress)) + { + PsfTrace($" / {logicalAddress}"); + return true; + } + logicalAddress = this.PsfKeyAccessor.GetPreviousAddress(physicalAddress); + if (logicalAddress < hlog.HeadAddress) + break; // RECORD_ON_DISK or not found + physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + } + PsfTrace($"/{logicalAddress}"); + return false; + } + + [Conditional("PSF_TRACE")] + private void PsfTrace(string message) + { + if (!this.ImplmentsPSFs) Console.Write(message); + } + + [Conditional("PSF_TRACE")] + private void PsfTraceLine(string message = null) + { + if (this.ImplmentsPSFs) Console.WriteLine(message ?? string.Empty); + } + + // PsfKeyContainer is necessary because VarLenBlittableAllocator.GetKeyContainer will use the size of the full + // composite key (KeyPointerSize * PsfCount), but the query key has only one KeyPointer. + private class PsfQueryKeyContainer : IHeapContainer + { + private readonly SectorAlignedMemory mem; + + public unsafe PsfQueryKeyContainer(ref Key key, KeyAccessor keyAccessor, SectorAlignedBufferPool pool) + { + var len = keyAccessor.KeyPointerSize; + this.mem = pool.Get(len); + Buffer.MemoryCopy(Unsafe.AsPointer(ref key), mem.GetValidPointer(), len, len); + } + + public unsafe ref Key Get() => ref Unsafe.AsRef(this.mem.GetValidPointer()); + + public void Dispose() => this.mem.Return(); + } + + private IPSFFunctions GetFunctions(ref Context context) + => (context as PSFContext).Functions as IPSFFunctions; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal OperationStatus PsfInternalReadKey( + ref Key queryKeyPointerRefAsKeyRef, ref Input input, ref Output output, ref Context context, + ref PendingContext pendingContext, + FasterSession fasterSession, + FasterExecutionContext sessionCtx, long lsn) + where FasterSession : IFasterSession + { + // Note: This function is called only for the secondary FasterKV. + var bucket = default(HashBucket*); + var slot = default(int); + var latestRecordVersion = -1; + var heldOperation = LatchOperation.None; + + var functions = GetFunctions(ref context); + ref KeyPointer queryKeyPointer = ref KeyPointer.CastFromKeyRef(ref queryKeyPointerRefAsKeyRef); + + var hash = this.PsfKeyAccessor.GetHashCode64(ref queryKeyPointer); + var tag = (ushort)((ulong)hash >> Constants.kHashTagShift); + + if (sessionCtx.phase != Phase.REST) + HeavyEnter(hash, sessionCtx, fasterSession); + + #region Trace back for record in in-memory HybridLog + HashBucketEntry entry = default; + var tagExists = FindTag(hash, tag, ref bucket, ref slot, ref entry); + OperationStatus status; + + // For PSFs, the addresses stored in the hash table point to KeyPointer entries, not the record header. + PsfTrace($"ReadKey: {this.PsfKeyAccessor?.GetString(ref queryKeyPointer)} | hash {hash} |"); + long logicalAddress = Constants.kInvalidAddress; + if (tagExists) + { + logicalAddress = entry.Address; + PsfTrace($" {logicalAddress}"); + +#if false // TODOdcr: Support ReadCache in PSFs (must call this.PsfKeyAccessor.GetRecordAddressFromKeyLogicalAddress) + if (UseReadCache && ReadFromCache(ref queryKey, ref logicalAddress, ref physicalAddress, ref latestRecordVersion, psfInput)) + { + if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot thread + } + return functions.VisitReadCache(input, ref hlog.GetKey(physicalAddress), + ref readcache.GetValue(physicalAddress), + hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: false).Status; + } +#endif + + if (logicalAddress >= hlog.HeadAddress) + { + if (!ScanQueryChain(ref logicalAddress, ref queryKeyPointer, ref latestRecordVersion)) + goto ProcessAddress; // RECORD_ON_DISK or not found + } + } + else + { + PsfTraceLine($" 0"); + return OperationStatus.NOTFOUND; // no tag found + } +#endregion + + if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) + { + PsfTraceLine("CPR_SHIFT_DETECTED"); + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot thread + } + + #region Normal processing + + ProcessAddress: + PsfTraceLine(); + if (logicalAddress >= hlog.HeadAddress) + { + // Mutable region (even fuzzy region is included here) is above SafeReadOnlyAddress and + // is concurrent; Immutable region will not be changed. + long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + long recordAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + return functions.VisitSecondaryRead(ref hlog.GetValue(recordAddress), ref input, ref output, physicalAddress, + hlog.GetInfo(recordAddress).Tombstone, + isConcurrent: logicalAddress >= hlog.SafeReadOnlyAddress).Status; + } + + // On-Disk Region + else if (logicalAddress >= hlog.BeginAddress) + { + status = OperationStatus.RECORD_ON_DISK; + if (sessionCtx.phase == Phase.PREPARE) + { + Debug.Assert(heldOperation != LatchOperation.Exclusive); + if (heldOperation == LatchOperation.Shared || HashBucket.TryAcquireSharedLatch(bucket)) + heldOperation = LatchOperation.Shared; + else + status = OperationStatus.CPR_SHIFT_DETECTED; + + if (RelaxedCPR) // don't hold on to shared latched during IO + { + if (heldOperation == LatchOperation.Shared) + HashBucket.ReleaseSharedLatch(bucket); + heldOperation = LatchOperation.None; + } + } + goto CreatePendingContext; + } + else + { + // No record found + return OperationStatus.NOTFOUND; + } + +#endregion + +#region Create pending context + CreatePendingContext: + { + pendingContext.type = OperationType.PSF_READ_KEY; + pendingContext.key = new PsfQueryKeyContainer(ref queryKeyPointerRefAsKeyRef, this.PsfKeyAccessor, this.hlog.bufferPool); + pendingContext.input = input; + pendingContext.output = output; + pendingContext.userContext = default; + pendingContext.entry.word = entry.word; + pendingContext.logicalAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(hlog.GetPhysicalAddress(logicalAddress)); + pendingContext.version = sessionCtx.version; + pendingContext.serialNum = lsn; + pendingContext.heldLatch = heldOperation; + } +#endregion + + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal OperationStatus PsfInternalReadAddress( + ref Input input, ref Output output, ref Context context, + ref PendingContext pendingContext, + FasterSession fasterSession, + FasterExecutionContext sessionCtx, long lsn) + where FasterSession : IFasterSession + { + // Notes: + // - This function is called for both the primary and secondary FasterKV. + // - Because we are retrieving a specific address rather than looking up by key, we are not in a position + // to scan for a particular record version--and thus do not consider CPR boundaries, so latestRecordVersion + // is used only as a target for ScanQueryChain. + // TODO: Support a variation of this that allows traversing from a start address -or- the hash table, and returns next start address. + var latestRecordVersion = -1; + OperationStatus status; + var functions = GetFunctions(ref context); + + #region Look up record in in-memory HybridLog + // For PSFs, the addresses stored in the hash table point to KeyPointer entries, not the record header. + long logicalAddress = functions.ReadLogicalAddress(ref input); + PsfTrace($" ReadAddr: | {logicalAddress}"); + +#if false // TODOdcr: Support ReadCache in PSFs (must call this.PsfKeyAccessor.GetRecordAddressFromKeyLogicalAddress) + // TODO: PsfInternalReadAddress should handle ReadCache for primary FKV + if (UseReadCache && ReadFromCache(ref logicalAddress, ref physicalAddress, ref latestRecordVersion)) + { + if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) + { + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot thread + } + return psfOutput.Visit(psfInput.PsfOrdinal, ref hlog.GetKey(physicalAddress), + ref readcache.GetValue(physicalAddress), + hlog.GetInfo(physicalAddress).Tombstone, isConcurrent: false).Status; + } +#endif + + if (logicalAddress >= hlog.HeadAddress) + { + if (this.ImplmentsPSFs && !ScanQueryChain(ref logicalAddress, ref KeyPointer.CastFromKeyRef(ref functions.QueryKeyRef(ref input)), ref latestRecordVersion)) + { + goto ProcessAddress; // RECORD_ON_DISK or not found + } + } +#endregion + +#region Normal processing + + ProcessAddress: + PsfTraceLine(); + if (logicalAddress >= hlog.HeadAddress) + { + // Mutable region (even fuzzy region is included here) is above SafeReadOnlyAddress and + // is concurrent; Immutable region will not be changed. + long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + if (this.ImplmentsPSFs) + { + long recordAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + return functions.VisitSecondaryRead(ref hlog.GetValue(recordAddress), ref input, ref output, physicalAddress, + hlog.GetInfo(recordAddress).Tombstone, + isConcurrent: logicalAddress >= hlog.SafeReadOnlyAddress).Status; + } + return functions.VisitPrimaryReadAddress(ref hlog.GetKey(physicalAddress), ref hlog.GetValue(physicalAddress), + ref output, isConcurrent: logicalAddress >= hlog.SafeReadOnlyAddress).Status; + } + + // On-Disk Region + else if (logicalAddress >= hlog.BeginAddress) + { + // As mentioned above, we do not have a key here, so we do not worry about CPR and getting the hash, latching, etc. + status = OperationStatus.RECORD_ON_DISK; + goto CreatePendingContext; + } + else + { + // No record found. TODOerr: we should not have called this function in this case. + return OperationStatus.NOTFOUND; + } + +#endregion + +#region Create pending context + CreatePendingContext: + { + pendingContext.type = OperationType.PSF_READ_ADDRESS; + pendingContext.key = this.ImplmentsPSFs + ? new PsfQueryKeyContainer(ref functions.QueryKeyRef(ref input), this.PsfKeyAccessor, this.hlog.bufferPool) + : default; + pendingContext.input = input; + pendingContext.output = output; + pendingContext.userContext = default; + pendingContext.entry.word = default; + pendingContext.logicalAddress = this.ImplmentsPSFs + ? this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(hlog.GetPhysicalAddress(logicalAddress)) + : logicalAddress; + pendingContext.version = sessionCtx.version; + pendingContext.serialNum = lsn; + pendingContext.heldLatch = LatchOperation.None; + } +#endregion + + return status; + } + + unsafe struct CASHelper + { + internal HashBucket* bucket; + internal HashBucketEntry entry; + internal long hash; + internal int slot; + internal bool isNull; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal OperationStatus PsfInternalInsert( + ref Key firstKeyPointerRefAsKeyRef, ref Value value, ref Input input, ref Context context, + ref PendingContext pendingContext, + FasterSession fasterSession, + FasterExecutionContext sessionCtx, long lsn) + where FasterSession : IFasterSession + { + var status = default(OperationStatus); + var latestRecordVersion = -1; + + var functions = GetFunctions(ref context); + ref CompositeKey compositeKey = ref CompositeKey.CastFromFirstKeyPointerRefAsKeyRef(ref firstKeyPointerRefAsKeyRef); + + // Update the KeyPointer links for chains with IsNullAt false (indicating a match with the + // corresponding PSF) to point to the previous records for all keys in the composite key. + // Note: We're not checking for a previous occurrence of the input value (the recordId) because + // we are doing insert only here; the update part of upsert is done in PsfInternalUpdate. + var psfCount = this.PsfKeyAccessor.KeyCount; + CASHelper* casHelpers = stackalloc CASHelper[psfCount]; + int startOfKeysOffset = 0; + PsfTrace($"Insert: {this.PsfKeyAccessor.GetString(ref compositeKey)} | rId {value} |"); + for (var psfOrdinal = 0; psfOrdinal < psfCount; ++psfOrdinal) + { + // For RCU, or in case we had to retry due to CPR_SHIFT and somehow managed to delete + // the previously found record, clear out the chain link pointer. + this.PsfKeyAccessor.SetPreviousAddress(ref compositeKey, psfOrdinal, Constants.kInvalidAddress); + + this.PsfKeyAccessor.SetOffsetToStartOfKeys(ref compositeKey, psfOrdinal, startOfKeysOffset); + startOfKeysOffset += this.PsfKeyAccessor.KeyPointerSize; + + ref CASHelper casHelper = ref casHelpers[psfOrdinal]; + if (this.PsfKeyAccessor.IsNullAt(ref compositeKey, psfOrdinal)) + { + casHelper.isNull = true; + PsfTrace($" null"); + continue; + } + + casHelper.hash = this.PsfKeyAccessor.GetHashCode64(ref compositeKey, psfOrdinal); + var tag = (ushort)((ulong)casHelper.hash >> Constants.kHashTagShift); + + if (sessionCtx.phase != Phase.REST) + HeavyEnter(casHelper.hash, sessionCtx, fasterSession); + +#region Look up record in in-memory HybridLog + FindOrCreateTag(casHelper.hash, tag, ref casHelper.bucket, ref casHelper.slot, ref casHelper.entry, hlog.BeginAddress); + + // For PSFs, the addresses stored in the hash table point to KeyPointer entries, not the record header. + var logicalAddress = casHelper.entry.Address; + if (logicalAddress >= hlog.BeginAddress) + { + PsfTrace($" {logicalAddress}"); + + if (logicalAddress < hlog.BeginAddress) + continue; + + if (logicalAddress >= hlog.HeadAddress) + { + // Note that we do not backtrace here because we are not replacing the value at the key; + // instead, we insert at the top of the hash chain. Track the latest record version we've seen. + long physicalAddress = hlog.GetPhysicalAddress(logicalAddress); + var recordAddress = this.PsfKeyAccessor.GetRecordAddressFromKeyPhysicalAddress(physicalAddress); + if (hlog.GetInfo(physicalAddress).Tombstone) + { + // The chain might extend past a tombstoned record so we must include it in the chain + // unless its prevLink at psfOrdinal is invalid. + var prevAddress = this.PsfKeyAccessor.GetPreviousAddress(physicalAddress); + if (prevAddress < hlog.BeginAddress) + continue; + } + latestRecordVersion = Math.Max(latestRecordVersion, hlog.GetInfo(recordAddress).Version); + } + + this.PsfKeyAccessor.SetPreviousAddress(ref compositeKey, psfOrdinal, logicalAddress); + } + else + { + PsfTrace($" 0"); + } +#endregion + } + +#region Entry latch operation + // No actual checkpoint locking will be done because this is Insert; only the current thread can write to + // the record we're about to create, and no readers can see it until it is successfully inserted. However, we + // must pivot and retry any insertions if we have seen a later version in any record in the hash table. + if (sessionCtx.phase == Phase.PREPARE && latestRecordVersion != -1 && latestRecordVersion > sessionCtx.version) + { + PsfTraceLine("CPR_SHIFT_DETECTED"); + status = OperationStatus.CPR_SHIFT_DETECTED; + goto CreatePendingContext; // Pivot Thread + } + Debug.Assert(latestRecordVersion <= sessionCtx.version); + goto CreateNewRecord; +#endregion + +#region Create new record in the mutable region + CreateNewRecord: + { + // Create the new record. Because we are updating multiple hash buckets, mark the record as invalid to start, + // so it is not visible until we have successfully updated all chains. + var recordSize = hlog.GetRecordSize(ref firstKeyPointerRefAsKeyRef, ref value); + BlockAllocate(recordSize, out long newLogicalAddress, sessionCtx, fasterSession); + var newPhysicalAddress = hlog.GetPhysicalAddress(newLogicalAddress); + RecordInfo.WriteInfo(ref hlog.GetInfo(newPhysicalAddress), sessionCtx.version, + final:true, tombstone: functions.IsDelete(ref input), invalidBit:true, + Constants.kInvalidAddress); // We manage all prev addresses within CompositeKey + ref Key storedFirstKeyPointerRefAsKeyRef = ref hlog.GetKey(newPhysicalAddress); + ref CompositeKey storedKey = ref CompositeKey.CastFromFirstKeyPointerRefAsKeyRef(ref storedFirstKeyPointerRefAsKeyRef); + hlog.ShallowCopy(ref firstKeyPointerRefAsKeyRef, ref storedFirstKeyPointerRefAsKeyRef); + hlog.ShallowCopy(ref value, ref hlog.GetValue(newPhysicalAddress)); + + PsfTraceLine(); + newLogicalAddress += RecordInfo.GetLength(); + for (var psfOrdinal = 0; psfOrdinal < psfCount; ++psfOrdinal, newLogicalAddress += this.PsfKeyAccessor.KeyPointerSize) + { + var casHelper = casHelpers[psfOrdinal]; + var tag = (ushort)((ulong)casHelper.hash >> Constants.kHashTagShift); + + PsfTrace($" ({psfOrdinal}): {casHelper.hash} {tag} | newLA {newLogicalAddress} | prev {casHelper.entry.word}"); + if (casHelper.isNull) + { + PsfTraceLine(" null"); + continue; + } + + var newEntry = default(HashBucketEntry); + newEntry.Tag = tag; + newEntry.Address = newLogicalAddress & Constants.kAddressMask; + newEntry.Pending = casHelper.entry.Pending; + newEntry.Tentative = false; + + var foundEntry = default(HashBucketEntry); + while (true) + { + // If we do not succeed on the exchange, another thread has updated the slot, or we have done so + // with a colliding hash value from earlier in the current record. As long as we satisfy the + // invariant that the chain points downward (to lower addresses), we can retry. + foundEntry.word = Interlocked.CompareExchange(ref casHelper.bucket->bucket_entries[casHelper.slot], + newEntry.word, casHelper.entry.word); + if (foundEntry.word == casHelper.entry.word) + break; + + if (foundEntry.word < newEntry.word) + { + PsfTrace($" / {foundEntry.Address}"); + casHelper.entry.word = foundEntry.word; + this.PsfKeyAccessor.SetPreviousAddress(ref storedKey, psfOrdinal, foundEntry.Address); + continue; + } + + // We can't satisfy the always-downward invariant, so leave the record marked Invalid and go + // around again to try inserting another record. + PsfTraceLine("RETRY_NOW"); + status = OperationStatus.RETRY_NOW; + goto LatchRelease; + } + + // Success for this PSF. + PsfTraceLine(" ins"); + hlog.GetInfo(newPhysicalAddress).Invalid = false; + } + + storedKey.ClearUpdateFlags(this.PsfKeyAccessor.KeyCount, this.PsfKeyAccessor.KeyPointerSize); + status = OperationStatus.SUCCESS; + goto LatchRelease; + } +#endregion + +#region Create pending context + CreatePendingContext: + { + pendingContext.type = OperationType.PSF_INSERT; + pendingContext.key = hlog.GetKeyContainer(ref firstKeyPointerRefAsKeyRef); // The Insert key has the full PsfCount of KeyPointers + pendingContext.value = hlog.GetValueContainer(ref value); + pendingContext.input = input; + pendingContext.userContext = default; + pendingContext.entry.word = default; + pendingContext.logicalAddress = Constants.kInvalidAddress; + pendingContext.version = sessionCtx.version; + pendingContext.serialNum = lsn; + } +#endregion + +#region Latch release + LatchRelease: + // No actual latching was done. +#endregion + + return status == OperationStatus.RETRY_NOW + ? PsfInternalInsert(ref firstKeyPointerRefAsKeyRef, ref value, ref input, ref context, ref pendingContext, fasterSession, sessionCtx, lsn) + : status; + } + } +} \ No newline at end of file diff --git a/cs/src/core/Index/PSF/FasterPSFLogOperations.cs b/cs/src/core/Index/PSF/FasterPSFLogOperations.cs new file mode 100644 index 000000000..deeb89beb --- /dev/null +++ b/cs/src/core/Index/PSF/FasterPSFLogOperations.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + public partial class FasterKV : FasterBase, IFasterKV + { + /// + public void FlushPSFLogs(bool wait) => this.PSFManager.FlushLogs(wait); + + /// + public void FlushAndEvictPSFLogs(bool wait) => this.PSFManager.FlushAndEvictLogs(wait); + + /// + public void DisposePSFLogsFromMemory() => this.PSFManager.DisposeLogsFromMemory(); + } +} diff --git a/cs/src/core/Index/PSF/FasterPSFRegistration.cs b/cs/src/core/Index/PSF/FasterPSFRegistration.cs new file mode 100644 index 000000000..69ba59afd --- /dev/null +++ b/cs/src/core/Index/PSF/FasterPSFRegistration.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Linq; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FASTER.core +{ + // PSF-related functions for FasterKV + public partial class FasterKV : FasterBase, IFasterKV + { + internal PSFManager, long> PSFManager { get; private set; } + + internal void InitializePSFManager() + => this.PSFManager = new PSFManager, long>(); + + #region PSF Registration API + /// + public IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, + FasterKVPSFDefinition def) + where TPSFKey : struct + => this.PSFManager.RegisterPSF(registrationSettings, def); + + /// + public IPSF[] RegisterPSF(PSFRegistrationSettings registrationSettings, + params FasterKVPSFDefinition[] defs) + where TPSFKey : struct + => this.PSFManager.RegisterPSF(registrationSettings, defs); + + /// + public IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, + string psfName, Func psfFunc) + where TPSFKey : struct + => this.PSFManager.RegisterPSF(registrationSettings, new FasterKVPSFDefinition(psfName, psfFunc)); + + /// + public IPSF[] RegisterPSF(PSFRegistrationSettings registrationSettings, + params (string, Func)[] psfFuncs) + where TPSFKey : struct + => this.PSFManager.RegisterPSF(registrationSettings, psfFuncs.Select(e => new FasterKVPSFDefinition(e.Item1, e.Item2)).ToArray()); + + /// + public string[][] GetRegisteredPSFNames() => this.PSFManager.GetRegisteredPSFNames(); + #endregion PSF Registration API + } +} diff --git a/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs new file mode 100644 index 000000000..aa82d5265 --- /dev/null +++ b/cs/src/core/Index/PSF/FasterPSFSessionOperations.cs @@ -0,0 +1,807 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace FASTER.core +{ + public sealed partial class ClientSession : IClientSession, IDisposable + where Functions : IFunctions + { + #region PSF calls for Secondary FasterKV + + // This value is created within the Primary FKV session. + Lazy, PSFContext, PSFPrimaryFunctions>> psfLookupRecordIdSession; + + internal void CreateLazyPsfSessionWrapper() { + this.psfLookupRecordIdSession = new Lazy, PSFContext, PSFPrimaryFunctions>>( + () => this.fht.NewSession, PSFContext, PSFPrimaryFunctions>( + new PSFPrimaryFunctions())); + } + + internal void DisposeLazyPsfSessionWrapper() + { + if (!(this.psfLookupRecordIdSession is null) && this.psfLookupRecordIdSession.IsValueCreated) + this.psfLookupRecordIdSession.Value.Dispose(); + } + + private ClientSession, PSFContext, PSFPrimaryFunctions> GetPsfLookupRecordSession() + => this.psfLookupRecordIdSession.Value; + + internal Status PsfInsert(ref Key key, ref Value value, ref Input input, ref Context context, long serialNo) + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return fht.ContextPsfInsert(ref key, ref value, ref input, ref context, this.FasterSession, serialNo, ctx); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + internal Status PsfReadKey(ref Key key, ref Input input, ref Output output, ref Context context, long serialNo) + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return fht.ContextPsfReadKey(ref key, ref input, ref output, ref context, this.FasterSession, serialNo, ctx); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + internal ValueTask.ReadAsyncResult> PsfReadKeyAsync( + ref Key key, ref Input input, ref Output output, ref Context context, long serialNo, PSFQuerySettings querySettings) + { + // Called on the secondary FasterKV + return fht.ContextPsfReadKeyAsync(this, ref key, ref input, ref output, ref context, serialNo, ctx, querySettings); + } + + internal Status PsfReadAddress(ref Input input, ref Output output, ref Context context, long serialNo) + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return fht.ContextPsfReadAddress(ref input, ref output, ref context, this.FasterSession, serialNo, ctx); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + internal ValueTask.ReadAsyncResult> PsfReadAddressAsync( + ref Input input, ref Output output, ref Context context, long serialNo, PSFQuerySettings querySettings) + { + // Called on the secondary FasterKV + return fht.ContextPsfReadAddressAsync(this, ref input, ref output, ref context, serialNo, ctx, querySettings); + } + + internal Status PsfUpdate(ref GroupCompositeKeyPair groupKeysPair, ref Value value, ref Input input, + ref Context context, long serialNo, + PSFChangeTracker changeTracker) + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return fht.ContextPsfUpdate(ref groupKeysPair, ref value, ref input, ref context, this.FasterSession, serialNo, ctx, changeTracker); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + internal Status PsfDelete(ref Key key, ref Value value, ref Input input, ref Context context, long serialNo, + PSFChangeTracker changeTracker) + { + // Called on the secondary FasterKV + if (SupportAsync) UnsafeResumeThread(); + try + { + return fht.ContextPsfDelete(ref key, ref value, ref input, ref context, this.FasterSession, serialNo, ctx, changeTracker); + } + finally + { + if (SupportAsync) UnsafeSuspendThread(); + } + } + + #endregion PSF calls for Secondary FasterKV + + #region PSF Query API for primary FasterKV + + internal Status CreateProviderData(long logicalAddress, ConcurrentQueue> providerDatas) + { + // Looks up logicalAddress in the primary FasterKV + var output = new PSFOutputPrimaryReadAddress(this.fht.hlog, providerDatas); + var input = new PSFInputPrimaryReadAddress(logicalAddress); + var session = this.GetPsfLookupRecordSession(); + var context = new PSFContext { Functions = session.functions }; + return session.PsfReadAddress(ref input, ref output, ref context, this.ctx.serialNum + 1); + } + + internal IEnumerable> ReturnProviderDatas(IEnumerable logicalAddresses) + { + // If the Primary FKV record is on disk the Read will go pending and we will not receive it "synchronously" + // here; instead, it will work its way through the pending read system and call psfOutput.Visit. + // providerDatas gives that a place to put the record. We should encounter this only after all + // non-pending records have been read, but this approach allows any combination of pending and + // non-pending reads. + var providerDatas = new ConcurrentQueue>(); + foreach (var logicalAddress in logicalAddresses) + { + var status = this.CreateProviderData(logicalAddress, providerDatas); + if (status == Status.ERROR) + { + // TODOerr: Handle error status from PsfReadAddress + } + while (providerDatas.TryDequeue(out var providerData)) + { + // TODO: Liveness check via Read + yield return providerData; + } + } + + this.CompletePending(spinWait: true); + while (providerDatas.TryDequeue(out var providerData)) + yield return providerData; + } + +#if DOTNETCORE + internal async ValueTask> CreateProviderDataAsync(long logicalAddress, ConcurrentQueue> providerDatas, PSFQuerySettings querySettings) + { + // Looks up logicalAddress in the primary FasterKV + var output = new PSFOutputPrimaryReadAddress(this.fht.hlog, providerDatas); + var input = new PSFInputPrimaryReadAddress(logicalAddress); + var session = this.GetPsfLookupRecordSession(); + var context = new PSFContext { Functions = session.functions }; + var readAsyncResult = await session.PsfReadAddressAsync(ref input, ref output, ref context, this.ctx.serialNum + 1, querySettings); + if (querySettings.IsCanceled) + return null; + var (status, _) = readAsyncResult.CompleteRead(); + if (status != Status.OK) // TODOerr: check other status + return null; + return providerDatas.TryDequeue(out var providerData) ? providerData : null; + } + + internal async IAsyncEnumerable> ReturnProviderDatasAsync(IAsyncEnumerable logicalAddresses, PSFQuerySettings querySettings) + { + querySettings ??= PSFQuerySettings.Default; + + // For the async form, we always read fully; there is no pending. + var providerDatas = new ConcurrentQueue>(); + await foreach (var logicalAddress in logicalAddresses) + { + var providerData = await this.CreateProviderDataAsync(logicalAddress, providerDatas, querySettings); + { + // TODO: No Async query ops if threadaffinitized + // TODO: Liveness check via ReadAsync + yield return providerData; + } + } + } +#endif + + /// + /// Issue a query on a single on a single key value. + /// + /// + /// foreach (var providerData in fht.QueryPSF(sizePsf, Size.Medium)) {...} + /// + /// The type of the key value to return results for + /// The Predicate Subset Function object + /// The key value to return results for + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + IPSF psf, TPSFKey key, PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, key, querySettings)); + } + +#if DOTNETCORE + /// + /// Issue a query on a single on a single key value. + /// + /// + /// foreach (var providerData in fht.QueryPSF(sizePsf, Size.Medium)) {...} + /// + /// The type of the key value to return results for + /// The Predicate Subset Function object + /// The key value to return results for + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IPSF psf, TPSFKey key, PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psf, key, querySettings), querySettings); + } +#endif // DOTNETCORE + + /// + /// Issue a query on a single on multiple key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF(sizePsf, new TestPSFKey[] { Size.Medium, Size.Large })) {...} + /// (Note that this example requires an implicit TestPSFKey constructor taking Size). + /// + /// The type of the key value to return results for + /// The Predicate Subset Function object + /// A vector of key values to return results for; for example, an OR query on + /// a single PSF, or a range query for a PSF that generates keys identifying bins. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + IPSF psf, IEnumerable keys, PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf, keys, querySettings)); + } + +#if DOTNETCORE + /// + /// Issue a query on a single on multiple key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF(sizePsf, new TestPSFKey[] { Size.Medium, Size.Large })) {...} + /// (Note that this example requires an implicit TestPSFKey constructor taking Size). + /// + /// The type of the key value to return results for + /// The Predicate Subset Function object + /// A vector of key values to return results for; for example, an OR query on + /// a single PSF, or a range query for a PSF that generates keys identifying bins. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IPSF psf, IEnumerable keys, PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psf, keys, querySettings), querySettings); + } +#endif // DOTNETCORE + + /// + /// Issue a query on two s, each with a single key value. + /// + /// + /// var providerData in fht.QueryPSF(sizePsf, Size.Medium, colorPsf, Color.Red, (l, r) => l || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The first Predicate Subset Function object + /// The second Predicate Subset Function object + /// The key value to return results from the first 's stored values + /// The key value to return results from the second 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf1, key1, psf2, key2, matchPredicate, querySettings)); + } + +#if DOTNETCORE + /// + /// Issue a query on two s, each with a single key value. + /// + /// + /// var providerData in fht.QueryPSF(sizePsf, Size.Medium, colorPsf, Color.Red, (l, r) => l || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The first Predicate Subset Function object + /// The second Predicate Subset Function object + /// The key value to return results from the first 's stored values + /// The key value to return results from the second 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psf1, key1, psf2, key2, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + + /// + /// Issue a query on two s, each with a vector of key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// sizePsf, new [] { new SizeKey(Size.Medium), new SizeKey(Size.Large) }, + /// colorPsf, new [] { new ColorKey(Color.Red), new ColorKey(Color.Blue) }, + /// (l, r) => l || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The first Predicate Subset Function object + /// The secojnd Predicate Subset Function object + /// The key values to return results from the first 's stored values + /// The key values to return results from the second 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf1, keys1, psf2, keys2, matchPredicate, querySettings)); + } + +#if DOTNETCORE + /// + /// Issue a query on two s, each with a vector of key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// sizePsf, new [] { new SizeKey(Size.Medium), new SizeKey(Size.Large) }, + /// colorPsf, new [] { new ColorKey(Color.Red), new ColorKey(Color.Blue) }, + /// (l, r) => l || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The first Predicate Subset Function object + /// The secojnd Predicate Subset Function object + /// The key values to return results from the first 's stored values + /// The key values to return results from the second 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psf1, keys1, psf2, keys2, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + + /// + /// Issue a query on three s, each with a single key value. + /// + /// + /// var providerData in fht.QueryPSF(sizePsf, Size.Medium, colorPsf, Color.Red, countPsf, 7, (l, m, r) => l || m || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The type of the key value for the third + /// The first Predicate Subset Function object + /// The second Predicate Subset Function object + /// The third Predicate Subset Function object + /// The key value to return results from the first 's stored values + /// The key value to return results from the second 's stored values + /// The key value to return results from the third 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, 3) whether the record matches the third PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + IPSF psf3, TPSFKey3 key3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf1, key1, psf2, key2, psf3, key3, matchPredicate, querySettings)); + } + +#if DOTNETCORE + /// + /// Issue a query on three s, each with a single key value. + /// + /// + /// var providerData in fht.QueryPSF(sizePsf, Size.Medium, colorPsf, Color.Red, countPsf, 7, (l, m, r) => l || m || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The type of the key value for the third + /// The first Predicate Subset Function object + /// The second Predicate Subset Function object + /// The third Predicate Subset Function object + /// The key value to return results from the first 's stored values + /// The key value to return results from the second 's stored values + /// The key value to return results from the third 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, 3) whether the record matches the third PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + IPSF psf3, TPSFKey3 key3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psf1, key1, psf2, key2, psf3, key3, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + + /// + /// Issue a query on three s, each with a vector of key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// sizePsf, new [] { new SizeKey(Size.Medium), new SizeKey(Size.Large) }, + /// colorPsf, new [] { new ColorKey(Color.Red), new ColorKey(Color.Blue) }, + /// countPsf, new [] { new CountKey(7), new CountKey(42) }, + /// (l, m, r) => l || m || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The type of the key value for the third + /// The first Predicate Subset Function object + /// The second Predicate Subset Function object + /// The third Predicate Subset Function object + /// The key values to return results from the first 's stored values + /// The key values to return results from the second 's stored values + /// The key values to return results from the third 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, 3) whether the record matches the third PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + IPSF psf3, IEnumerable keys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psf1, keys1, psf2, keys2, psf3, keys3, matchPredicate, querySettings)); + } + +#if DOTNETCORE + /// + /// Issue a query on three s, each with a vector of key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// sizePsf, new [] { new SizeKey(Size.Medium), new SizeKey(Size.Large) }, + /// colorPsf, new [] { new ColorKey(Color.Red), new ColorKey(Color.Blue) }, + /// countPsf, new [] { new CountKey(7), new CountKey(42) }, + /// (l, m, r) => l || m || r)) + /// + /// The type of the key value for the first + /// The type of the key value for the second + /// The type of the key value for the third + /// The first Predicate Subset Function object + /// The second Predicate Subset Function object + /// The third Predicate Subset Function object + /// The key values to return results from the first 's stored values + /// The key values to return results from the second 's stored values + /// The key values to return results from the third 's stored values + /// A predicate that takes as parameters 1) whether a candidate record matches + /// the first PSF, 2) whether the record matches the second PSF, 3) whether the record matches the third PSF, and returns a bool indicating whether the + /// record should be part of the result set. For example, an AND query would return true iff both input + /// parameters are true, else false; an OR query would return true if either input parameter is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + IPSF psf3, IEnumerable keys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psf1, keys1, psf2, keys2, psf3, keys3, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + + /// + /// Issue a query on one or more s, each with a vector of key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// new[] { + /// (sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }), + /// (colorPsf, new TestPSFKey[] { Color.Red, Color.Blue})}, + /// ll => ll[0])) + /// (Note that this example requires an implicit TestPSFKey constructor taking Size). + /// + /// The type of the key value for the vector + /// A vector of s and associated keys to be queried + /// A predicate that takes as a parameters a boolean vector in parallel with + /// the vector indicating whether a candidate record matches the corresponding + /// , and returns a bool indicating whether the record should be part of + /// the result set. For example, an AND query would return true iff all elements of the input vector are true, + /// else false; an OR query would return true if element of the input vector is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psfsAndKeys, matchPredicate, querySettings)); + } + +#if DOTNETCORE + /// + /// Issue a query on one or more s, each with a vector of key values. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// new[] { + /// (sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }), + /// (colorPsf, new TestPSFKey[] { Color.Red, Color.Blue})}, + /// ll => ll[0])) + /// (Note that this example requires an implicit TestPSFKey constructor taking Size). + /// + /// The type of the key value for the vector + /// A vector of s and associated keys to be queried + /// A predicate that takes as a parameters a boolean vector in parallel with + /// the vector indicating whether a candidate record matches the corresponding + /// , and returns a bool indicating whether the record should be part of + /// the result set. For example, an AND query would return true iff all elements of the input vector are true, + /// else false; an OR query would return true if element of the input vector is true. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psfsAndKeys, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + + /// + /// Issue a query on multiple keys s for two different key types. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// new[] { + /// (sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }), + /// (colorPsf, new TestPSFKey[] { Color.Red, Color.Blue })}, + /// new[] { + /// (countPsf, new [] { new CountKey(7), new CountKey(9) })}, + /// (ll, rr) => ll[0] || rr[0])) + /// (Note that this example requires an implicit TestPSFKey constructor taking Size). + /// + /// The type of the key value for the first vector's s + /// The type of the key value for the second vector's s + /// A vector of s and associated keys + /// of type to be queried + /// A vector of s and associated keys + /// of type to be queried + /// A predicate that takes as a parameters a boolean vector in parallel with + /// the vector and a second boolean vector in parallel with + /// the vector, and returns a bool indicating whether the record should be part of + /// the result set. For example, an AND query would return true iff all elements of both input vectors are true, + /// else false; an OR query would return true if any element of either input vector is true; and more complex + /// logic could be done depending on the specific PSFs. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psfsAndKeys1, psfsAndKeys2, matchPredicate, querySettings)); + } + +#if DOTNETCORE + /// + /// Issue a query on multiple keys s for two different key types. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// new[] { + /// (sizePsf, new TestPSFKey[] { Size.Medium, Size.Large }), + /// (colorPsf, new TestPSFKey[] { Color.Red, Color.Blue })}, + /// new[] { + /// (countPsf, new [] { new CountKey(7), new CountKey(9) })}, + /// (ll, rr) => ll[0] || rr[0])) + /// (Note that this example requires an implicit TestPSFKey constructor taking Size). + /// + /// The type of the key value for the first vector's s + /// The type of the key value for the second vector's s + /// A vector of s and associated keys + /// of type to be queried + /// A vector of s and associated keys + /// of type to be queried + /// A predicate that takes as a parameters a boolean vector in parallel with + /// the vector and a second boolean vector in parallel with + /// the vector, and returns a bool indicating whether the record should be part of + /// the result set. For example, an AND query would return true iff all elements of both input vectors are true, + /// else false; an OR query would return true if any element of either input vector is true; and more complex + /// logic could be done depending on the specific PSFs. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psfsAndKeys1, psfsAndKeys2, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + + /// + /// Issue a query on multiple keys s for three different key types. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// new[] { (sizePsf, new [] { new SizeKey(Size.Medium), new SizeKey(Size.Large) }) }, + /// new[] { (colorPsf, new [] { new ColorKey(Color.Red), new ColorKey(Color.Blue) }) }, + /// new[] { (countPsf, new [] { new CountKey(4), new CountKey(7) }) }, + /// (ll, mm, rr) => ll[0] || mm[0] || rr[0])) + /// + /// The type of the key value for the first vector's s + /// The type of the key value for the second vector's s + /// The type of the key value for the third vector's s + /// A vector of s and associated keys + /// of type to be queried + /// A vector of s and associated keys + /// of type to be queried + /// A vector of s and associated keys + /// of type to be queried + /// A predicate that takes as a parameters three boolean vectors in parallel with + /// each other, and returns a bool indicating whether the record should be part of + /// the result set. For example, an AND query would return true iff all elements of all input vectors are true, + /// else false; an OR query would return true if any element of either input vector is true; and more complex + /// logic could be done depending on the specific PSFs. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IEnumerable> QueryPSF( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatas(this.fht.PSFManager.QueryPSF(psfsAndKeys1, psfsAndKeys2, psfsAndKeys3, matchPredicate, querySettings)); + } + +#if DOTNETCORE + /// + /// Issue a query on multiple keys s for three different key types. + /// + /// + /// foreach (var providerData in fht.QueryPSF( + /// new[] { (sizePsf, new [] { new SizeKey(Size.Medium), new SizeKey(Size.Large) }) }, + /// new[] { (colorPsf, new [] { new ColorKey(Color.Red), new ColorKey(Color.Blue) }) }, + /// new[] { (countPsf, new [] { new CountKey(4), new CountKey(7) }) }, + /// (ll, mm, rr) => ll[0] || mm[0] || rr[0])) + /// + /// The type of the key value for the first vector's s + /// The type of the key value for the second vector's s + /// The type of the key value for the third vector's s + /// A vector of s and associated keys + /// of type to be queried + /// A vector of s and associated keys + /// of type to be queried + /// A vector of s and associated keys + /// of type to be queried + /// A predicate that takes as a parameters three boolean vectors in parallel with + /// each other, and returns a bool indicating whether the record should be part of + /// the result set. For example, an AND query would return true iff all elements of all input vectors are true, + /// else false; an OR query would return true if any element of either input vector is true; and more complex + /// logic could be done depending on the specific PSFs. + /// Optional query settings for EOS, cancellation, etc. + /// An enumerable of the FasterKV-specific provider data from the primary FasterKV + /// instance, as identified by the TRecordIds stored in the secondary FasterKV instances + public IAsyncEnumerable> QueryPSFAsync( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + // Unsafe(Resume|Suspend)Thread are done in the session.PsfRead* operations called by PSFGroup.QueryPSF. + return this.ReturnProviderDatasAsync(this.fht.PSFManager.QueryPSFAsync(psfsAndKeys1, psfsAndKeys2, psfsAndKeys3, matchPredicate, querySettings), querySettings); + } +#endif // DOTNETCORE + + #endregion PSF Query API for primary FasterKV + } +} diff --git a/cs/src/core/Index/PSF/GroupCompositeKey.cs b/cs/src/core/Index/PSF/GroupCompositeKey.cs new file mode 100644 index 000000000..630b51013 --- /dev/null +++ b/cs/src/core/Index/PSF/GroupCompositeKey.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + internal struct GroupCompositeKey : IDisposable + { + // This cannot be typed to a TPSFKey because there may be different TPSFKeys across groups. + private SectorAlignedMemory KeyPointerMem; + + internal void Set(SectorAlignedMemory keyMem) => this.KeyPointerMem = keyMem; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal unsafe ref TCompositeOrIndividualKey CastToKeyRef() + => ref Unsafe.AsRef(this.KeyPointerMem.GetValidPointer()); + + public void Dispose() => this.KeyPointerMem?.Return(); + } + + internal struct GroupCompositeKeyPair : IDisposable + { + internal long GroupId; + + // If the PSFGroup found the RecordId in its IPUCache, we carry it here. + internal long LogicalAddress; + + internal GroupCompositeKey Before; + internal GroupCompositeKey After; + + internal GroupCompositeKeyPair(long id) + { + this.GroupId = id; + this.LogicalAddress = Constants.kInvalidAddress; + this.Before = default; + this.After = default; + this.HasChanges = false; + } + + internal bool HasAddress => this.LogicalAddress != Constants.kInvalidAddress; + + internal ref TCompositeKey GetBeforeKey() => ref this.Before.CastToKeyRef(); + + internal ref TCompositeKey GetAfterKey() => ref this.After.CastToKeyRef(); + + internal bool HasChanges; + + public void Dispose() + { + this.Before.Dispose(); + this.After.Dispose(); + } + } +} diff --git a/cs/src/core/Index/PSF/IExecutePSF.cs b/cs/src/core/Index/PSF/IExecutePSF.cs new file mode 100644 index 000000000..258b5c7af --- /dev/null +++ b/cs/src/core/Index/PSF/IExecutePSF.cs @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace FASTER.core +{ + /// + /// This interface is implemented on a and decouples + /// the execution from the knowledge of the TKVKey and TKVValue of the + /// primary FasterKV instance. + /// + /// + /// + public interface IExecutePSF + where TRecordId : struct + { + /// + /// For each in the , + /// and store the resultant TPSFKey in the secondary FasterKV instance. + /// + /// The provider's data, e.g. + /// The provider's record ID, e.g. long (logicalAddress) for FasterKV + /// The phase of PSF operations in which this execution is being done + /// Tracks the values for comparison + /// to the values + Status ExecuteAndStore(TProviderData data, TRecordId recordId, PSFExecutePhase phase, + PSFChangeTracker changeTracker); + + /// + /// The identifier of this . + /// + long Id { get; } + + /// + /// Get the TPSFKeys for the current (before updating) state of the RecordId + /// The record of previous key values and updated values + /// + Status GetBeforeKeys(PSFChangeTracker changeTracker); + + /// + /// Update the RecordId + /// The record of previous key values and updated values + /// + Status Update(PSFChangeTracker changeTracker); + + /// + /// Delete the RecordId + /// The record of previous key values and updated values + /// + Status Delete(PSFChangeTracker changeTracker); + + /// + /// Take a full checkpoint of the FasterKV implementing the group's PSFs. + /// + bool TakeFullCheckpoint(); + + /// + /// Complete ongoing checkpoint (spin-wait) + /// + ValueTask CompleteCheckpointAsync(CancellationToken token = default); + + /// + /// Take a checkpoint of the Index (hashtable) only + /// + bool TakeIndexCheckpoint(); + + /// + /// Take a checkpoint of the hybrid log only + /// + bool TakeHybridLogCheckpoint(); + + /// + /// Recover from last successful checkpoints + /// + void Recover(); + + /// + /// Flush PSF logs until current tail (records are still retained in memory) + /// + /// Synchronous wait for operation to complete + void FlushLog(bool wait); + + /// + /// Flush PSF logs and evict all records from memory + /// + /// Synchronous wait for operation to complete + /// When wait is false, this tells whether the full eviction was successfully registered with FASTER + public void FlushAndEvictLog(bool wait); + + /// + /// Delete PSF logs entirely from memory. Cannot allocate on the log + /// after this point. This is a synchronous operation. + /// + public void DisposeLogFromMemory(); + } +} diff --git a/cs/src/core/Index/PSF/IPSF.cs b/cs/src/core/Index/PSF/IPSF.cs new file mode 100644 index 000000000..5d03bf0d2 --- /dev/null +++ b/cs/src/core/Index/PSF/IPSF.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// A base interface for to decouple the generic type parameters. + /// + public interface IPSF + { + /// + /// The name of the ; must be unique among all + /// s. + /// + string Name { get; } + } +} diff --git a/cs/src/core/Index/PSF/IPSFDefinition.cs b/cs/src/core/Index/PSF/IPSFDefinition.cs new file mode 100644 index 000000000..263f421c3 --- /dev/null +++ b/cs/src/core/Index/PSF/IPSFDefinition.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; + +namespace FASTER.core +{ + /// + /// The definition of a single PSF (Predicate Subset Function) + /// + /// The data from the provider that the PSF should be run over + /// The data from the provider that the PSF should be run over + public interface IPSFDefinition + where TPSFKey : struct + { + /// + /// The callback used to obtain a new TPFSKey for the ProviderData record for this PSF definition. + /// + /// The representation of the data that was written to the primary store + /// (e.g. Upsert in FasterKV). + /// Null if the record does not match the PSF, else the indexing key for the record + public TPSFKey? Execute(TProviderData record); + + /// + /// The Name of the PSF, assigned by the caller. Must be unique among PSFs in the group. It is + /// used by the caller to index PSFs in the group in a friendly way. + /// + public string Name { get; } + + } +} diff --git a/cs/src/core/Index/PSF/IQueryPSF.cs b/cs/src/core/Index/PSF/IQueryPSF.cs new file mode 100644 index 000000000..52ae6bdf0 --- /dev/null +++ b/cs/src/core/Index/PSF/IQueryPSF.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Collections.Generic; + +namespace FASTER.core +{ + /// + /// Provides an interface on the that decouples the + /// PSFGroup from the primary FasterKV's TKVKey and TKVValue. + /// + /// + /// + public interface IQueryPSF + { + /// + /// Issues a query on the specified to return s. + /// + /// The ordinal of the in this group + /// The key to query on to rertrieve the s. + /// Optional query settings for EOS, cancellation, etc. + /// + IEnumerable Query(int psfOrdinal, TPSFKey key, PSFQuerySettings querySettings); + +#if DOTNETCORE + /// + /// Issues a query on the specified to return s. + /// + /// The ordinal of the in this group + /// The key to query on to rertrieve the s. + /// Optional query settings for EOS, cancellation, etc. + /// + IAsyncEnumerable QueryAsync(int psfOrdinal, TPSFKey key, PSFQuerySettings querySettings); +#endif + } +} diff --git a/cs/src/core/Index/PSF/KeyAccessor.cs b/cs/src/core/Index/PSF/KeyAccessor.cs new file mode 100644 index 000000000..615bb3a19 --- /dev/null +++ b/cs/src/core/Index/PSF/KeyAccessor.cs @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Runtime.CompilerServices; +using System.Text; + +namespace FASTER.core +{ + /// + /// Provides access to the internals that are hidden behind + /// the Key typeparam of the secondary FasterKV. + /// + /// The type of the Key returned by a PSF function + internal unsafe class KeyAccessor + { + private readonly IFasterEqualityComparer userComparer; + + internal KeyAccessor(IFasterEqualityComparer userComparer, int keyCount, int keyPointerSize) + { + this.userComparer = userComparer; + this.KeyCount = keyCount; + this.KeyPointerSize = keyPointerSize; + } + + public int KeyCount { get; } + + public int KeyPointerSize { get; } + + #region KeyPointer accessors + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetPreviousAddress(long physicalAddress) + => this.GetKeyPointerRef(physicalAddress).PreviousAddress; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void SetPreviousAddress(ref CompositeKey key, int psfOrdinal, long prevAddress) + => this.GetKeyPointerRef(ref key, psfOrdinal).PreviousAddress = prevAddress; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void SetOffsetToStartOfKeys(ref CompositeKey key, int psfOrdinal, int offset) + => this.GetKeyPointerRef(ref key, psfOrdinal).OffsetToStartOfKeys = offset; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsNullAt(ref CompositeKey key, int psfOrdinal) => this.GetKeyPointerRef(ref key, psfOrdinal).IsNull; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsUnlinkOldAt(ref CompositeKey key, int psfOrdinal) => this.GetKeyPointerRef(ref key, psfOrdinal).IsUnlinkOld; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool IsLinkNewAt(ref CompositeKey key, int psfOrdinal) => this.GetKeyPointerRef(ref key, psfOrdinal).IsLinkNew; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetHashCode64(ref KeyPointer keyPointer) + => Utility.GetHashCode(this.userComparer.GetHashCode64(ref keyPointer.Key)) ^ Utility.GetHashCode(keyPointer.PsfOrdinal + 1); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetHashCode64(ref CompositeKey key, int psfOrdinal) + { + ref KeyPointer keyPointer = ref key.GetKeyPointerRef(psfOrdinal, this.KeyPointerSize); + return Utility.GetHashCode(this.userComparer.GetHashCode64(ref keyPointer.Key)) ^ Utility.GetHashCode(keyPointer.PsfOrdinal + 1); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool EqualsAtKeyAddress(ref KeyPointer queryKeyPointer, long physicalAddress) + => KeysEqual(ref queryKeyPointer, ref GetKeyPointerRef(physicalAddress)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool EqualsAtRecordAddress(ref KeyPointer queryKeyPointer, long physicalAddress) + => KeysEqual(ref queryKeyPointer, ref GetKeyPointerRef(physicalAddress, queryKeyPointer.PsfOrdinal)); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private bool KeysEqual(ref KeyPointer queryKeyPointer, ref KeyPointer storedKeyPointer) + => queryKeyPointer.PsfOrdinal == storedKeyPointer.PsfOrdinal && this.userComparer.Equals(ref queryKeyPointer.Key, ref storedKeyPointer.Key); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ref KeyPointer GetKeyPointerRef(ref CompositeKey key, int psfOrdinal) + => ref key.GetKeyPointerRef(psfOrdinal, this.KeyPointerSize); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal unsafe ref KeyPointer GetKeyPointerRef(long physicalAddress) + => ref Unsafe.AsRef>((byte*)physicalAddress); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal unsafe ref KeyPointer GetKeyPointerRef(long physicalAddress, int psfOrdinal) + => ref Unsafe.AsRef>((byte*)GetKeyAddressFromRecordPhysicalAddress(physicalAddress, psfOrdinal)); + #endregion KeyPointer accessors + + #region Address manipulation + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetRecordAddressFromKeyPhysicalAddress(long physicalAddress) + => physicalAddress - this.GetKeyPointerRef(physicalAddress).OffsetToStartOfKeys - RecordInfo.GetLength(); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long GetKeyAddressFromRecordPhysicalAddress(long physicalAddress, int psfOrdinal) + // TODOperf: if we omit IsNull keys, then this will have to walk to the key with psfOrdinal. Fortunately it is only + // called during AsyncGetFromDiskCallback. + => physicalAddress + RecordInfo.GetLength() + psfOrdinal * this.KeyPointerSize; + #endregion Address manipulation + + internal string GetString(ref CompositeKey compositeKey, int psfOrdinal = -1) + { + if (psfOrdinal == -1) + { + var sb = new StringBuilder("{"); + for (var ii = 0; ii < this.KeyCount; ++ii) + { + if (ii > 0) + sb.Append(", "); + ref KeyPointer keyPointer = ref this.GetKeyPointerRef(ref compositeKey, ii); + sb.Append(keyPointer.IsNull ? "null" : keyPointer.Key.ToString()); + } + sb.Append("}"); + return sb.ToString(); + } + return this.GetString(ref this.GetKeyPointerRef(ref compositeKey, psfOrdinal)); + } + + internal string GetString(ref KeyPointer keyPointer) + => $"{{{(keyPointer.IsNull ? "null" : keyPointer.Key.ToString())}}}"; + } +} \ No newline at end of file diff --git a/cs/src/core/Index/PSF/KeyPointer.cs b/cs/src/core/Index/PSF/KeyPointer.cs new file mode 100644 index 000000000..137945648 --- /dev/null +++ b/cs/src/core/Index/PSF/KeyPointer.cs @@ -0,0 +1,106 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + internal struct KeyPointer + { + #region Fields + /// + /// The previous address in the hash chain. May be for a different PsfOrdinal than this one due to hash collisions. + /// + internal long PreviousAddress; + + /// + /// The offset to the start of the record + /// + private ushort offsetToStartOfKeys; + + /// + /// The ordinal of the current . + /// + private byte psfOrdinal; // Note: 'byte' is consistent with Constants.kInvalidPsfOrdinal + + /// + /// Flags regarding the PSF. + /// + private byte flags; + + /// + /// The Key returned by the execution. + /// + internal TPSFKey Key; // TODOperf: for Key size > 4, reinterpret this an offset to the actual value (after the KeyPointer list) + #endregion Fields + + internal void Initialize(int psfOrdinal, ref TPSFKey key) + { + this.PreviousAddress = Constants.kInvalidAddress; + this.offsetToStartOfKeys = 0; + this.PsfOrdinal = psfOrdinal; + this.flags = 0; + this.Key = key; + } + + #region Accessors + // For Insert, this identifies a null PSF result (the record does not match the PSF and is not included + // in any TPSFKey chain for it). Also used in PSFChangeTracker to determine whether to set kUnlinkOldBit. + private const byte kIsNullBit = 0x01; + + // If Key size is > 4, then reinterpret the Key as an offset to the actual key. (TODOperf not implemented) + private const byte kIsOutOfLineKeyBit = 0x02; + + // For Update, the TPSFKey has changed; remove this record from the previous TPSFKey chain. + private const byte kUnlinkOldBit = 0x04; + + // For Update and insert, the TPSFKey has changed; add this record to the new TPSFKey chain. + private const byte kLinkNewBit = 0x08; + + internal bool IsNull + { + get => (this.flags & kIsNullBit) != 0; + set => this.flags = value ? (byte)(this.flags | kIsNullBit) : (byte)(this.flags & ~kIsNullBit); + } + + internal bool IsUnlinkOld + { + get => (this.flags & kUnlinkOldBit) != 0; + set => this.flags = value ? (byte)(this.flags | kUnlinkOldBit) : (byte)(this.flags & ~kUnlinkOldBit); + } + + internal bool IsLinkNew + { + get => (this.flags & kLinkNewBit) != 0; + set => this.flags = value ? (byte)(this.flags | kLinkNewBit) : (byte)(this.flags & ~kLinkNewBit); + } + + internal bool IsOutOfLineKey + { + get => (this.flags & kIsOutOfLineKeyBit) != 0; + set => this.flags = value ? (byte)(this.flags | kIsOutOfLineKeyBit) : (byte)(this.flags & ~kIsOutOfLineKeyBit); + } + + internal bool HasChanges => (this.flags & (kUnlinkOldBit | kLinkNewBit)) != 0; + + internal int PsfOrdinal + { + get => this.psfOrdinal; + set => this.psfOrdinal = (byte)value; + } + + internal int OffsetToStartOfKeys + { + get => this.offsetToStartOfKeys; + set => this.offsetToStartOfKeys = (ushort)value; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void ClearUpdateFlags() => this.flags = (byte)(this.flags & ~(kUnlinkOldBit | kLinkNewBit)); + #endregion Accessors + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal unsafe static ref KeyPointer CastFromKeyRef(ref TPSFKey keyRef) + => ref Unsafe.AsRef>((byte*)Unsafe.AsPointer(ref keyRef)); + } +} diff --git a/cs/src/core/Index/PSF/PSF.cs b/cs/src/core/Index/PSF/PSF.cs new file mode 100644 index 000000000..9cab5d535 --- /dev/null +++ b/cs/src/core/Index/PSF/PSF.cs @@ -0,0 +1,57 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + /// + /// The implementation of the Predicate Subset Function. + /// + /// The type of the key returned by the Predicate and store in the secondary + /// FasterKV instance + /// The type of data record supplied by the data provider; in FasterKV it + /// is the logicalAddress of the record in the primary FasterKV instance. + public class PSF : IPSF + { + private readonly IQueryPSF psfGroup; + + internal long GroupId { get; } // unique in the PSFManager.psfGroup list + + internal int PsfOrdinal { get; } // in the psfGroup + + // PSFs are passed by the caller to the session QueryPSF functions, so make sure they don't send + // a PSF from a different FKV. + internal Guid Id { get; } + + /// + public string Name { get; } + + internal PSF(long groupId, int psfOrdinal, string name, IQueryPSF iqp) + { + this.GroupId = groupId; + this.PsfOrdinal = psfOrdinal; + this.Name = name; + this.psfGroup = iqp; + this.Id = Guid.NewGuid(); + } + + /// + /// Issues a query on this PSF to return s. + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal IEnumerable Query(TPSFKey key, PSFQuerySettings querySettings) => this.psfGroup.Query(this.PsfOrdinal, key, querySettings); + +#if DOTNETCORE + /// + /// Issues a query on this PSF to return s. + /// + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal IAsyncEnumerable QueryAsync(TPSFKey key, PSFQuerySettings querySettings) => this.psfGroup.QueryAsync(this.PsfOrdinal, key, querySettings); +#endif + } +} diff --git a/cs/src/core/Index/PSF/PSFChangeTracker.cs b/cs/src/core/Index/PSF/PSFChangeTracker.cs new file mode 100644 index 000000000..33fcd3876 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFChangeTracker.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FASTER.core +{ + public enum UpdateOperation + { + Insert, + IPU, + RCU, + Delete + } + + public unsafe class PSFChangeTracker : IDisposable + // TODO: where TRecordId : struct + { + #region Data API + internal TProviderData BeforeData { get; private set; } + internal TRecordId BeforeRecordId { get; private set; } + + internal void SetBeforeData(TProviderData data, TRecordId recordId) + { + this.BeforeData = data; + this.BeforeRecordId = recordId; + } + + internal TProviderData AfterData { get; private set; } + internal TRecordId AfterRecordId { get; private set; } + + internal void SetAfterData(TProviderData data, TRecordId recordId) + { + this.AfterData = data; + this.AfterRecordId = recordId; + } + + public UpdateOperation UpdateOp { get; set; } + #endregion Data API + + private GroupCompositeKeyPair[] groups; + internal bool HasBeforeKeys { get; set; } + + internal long CachedBeforeLA = Constants.kInvalidAddress; + + internal PSFChangeTracker(IEnumerable groupIds) + { + this.groups = groupIds.Select(id => new GroupCompositeKeyPair(id)).ToArray(); + } + + internal bool FindGroup(long groupId, out int ordinal) + { + for (var ii = 0; ii < this.groups.Length; ++ii) // TODOperf: will there be enough groups for sequential search to matter? + { + if (groups[ii].GroupId == groupId) + { + ordinal = ii; + return true; + } + } + + // Likely the groupId was from a group added since this PSFChangeTracker instance was created. + ordinal = -1; + return false; + } + + internal ref GroupCompositeKeyPair GetGroupRef(int ordinal) => ref groups[ordinal]; + + internal ref GroupCompositeKeyPair FindGroupRef(long groupId, long logAddr = Constants.kInvalidAddress) + { + if (!this.FindGroup(groupId, out var ordinal)) + { + // A new group was added while we were populating this changeTracker; should be quite rare. // TODOtest: this case + var groups = new GroupCompositeKeyPair[this.groups.Length + 1]; + Array.Copy(this.groups, groups, this.groups.Length); + this.groups = groups; + ordinal = this.groups.Length - 1; + } + ref GroupCompositeKeyPair ret = ref this.groups[ordinal]; + ret.GroupId = groupId; + ret.LogicalAddress = logAddr; + return ref ret; + } + + public void Dispose() + { + foreach (var group in this.groups) + group.Dispose(); + } + } +} diff --git a/cs/src/core/Index/PSF/PSFContext.cs b/cs/src/core/Index/PSF/PSFContext.cs new file mode 100644 index 000000000..c2579bbe5 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFContext.cs @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// Context for operations on the secondary FasterKV instance. + /// + public class PSFContext + { + // // TODO Hack because we can't get the functions object from the session, so pass this as context (also hacked to make it a class) + internal IPSFFunctions Functions; + } +} diff --git a/cs/src/core/Index/PSF/PSFException.cs b/cs/src/core/Index/PSF/PSFException.cs new file mode 100644 index 000000000..7c10023af --- /dev/null +++ b/cs/src/core/Index/PSF/PSFException.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.Serialization; + +namespace FASTER.core +{ + /// + /// FASTER PSF exception base type + /// + public class PSFException : FasterException + { + internal PSFException() { } + + internal PSFException(string message) : base(message) { } + + internal PSFException(string message, Exception innerException) : base(message, innerException) { } + } + + /// + /// FASTER PSF argument exception type + /// + public class PSFArgumentException : PSFException + { + internal PSFArgumentException() { } + + internal PSFArgumentException(string message) : base(message) { } + + internal PSFArgumentException(string message, Exception innerException) : base(message, innerException) { } + } + + /// + /// FASTER PSF argument exception type + /// + public class PSFInvalidOperationException : PSFException + { + internal PSFInvalidOperationException() { } + + internal PSFInvalidOperationException(string message) : base(message) { } + + internal PSFInvalidOperationException(string message, Exception innerException) : base(message, innerException) { } + } + + /// + /// FASTER PSF argument exception type + /// + public class PSFInternalErrorException : PSFException + { + internal PSFInternalErrorException() { } + + internal PSFInternalErrorException(string message) : base($"Internal Error: {message}") { } + + internal PSFInternalErrorException(string message, Exception innerException) : base($"Internal Error: {message}", innerException) { } + } +} diff --git a/cs/src/core/Index/PSF/PSFExecutePhase.cs b/cs/src/core/Index/PSF/PSFExecutePhase.cs new file mode 100644 index 000000000..303094e53 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFExecutePhase.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// The phase of PSF operations in which PSFs are being executed + /// + public enum PSFExecutePhase + { + /// + /// Executing the PSFs to obtain new TPSFKeys to be inserted into the secondary FKV. + /// + Insert, + + /// + /// Lookup in IPUCache or execute the PSFs to obtain TPSFKeys prior to update, to be compared to those + /// obtained after the update to modify the record's PSF membership in the secondary FKV. + /// + PreUpdate, + + /// + /// Executing the PSFs to obtain TPSFKeys following an update, to be compared to those obtained before + /// the update to modify the record's PSF membership in the secondary FKV. + /// + PostUpdate, + + /// + /// Lookup in IPUCache to tombstone a record, or execute the PSFs to obtain TPSFKeys to place a new + /// tombstoned record. + /// + Delete + } +} diff --git a/cs/src/core/Index/PSF/PSFFunctions.cs b/cs/src/core/Index/PSF/PSFFunctions.cs new file mode 100644 index 000000000..e9e2ee248 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFFunctions.cs @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +using System.Diagnostics; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + // Non-generic interface + internal interface IPSFFunctions { } + + internal interface IPSFFunctions : IPSFFunctions + { + #region Input accessors + + long GroupId(ref TInput input); + + bool IsDelete(ref TInput input); + bool SetDelete(ref TInput input, bool value); + + public long ReadLogicalAddress(ref TInput input); + + public ref TKey QueryKeyRef(ref TInput input); + #endregion Input accessors + + #region Output visitors + PSFOperationStatus VisitPrimaryReadAddress(ref TKey key, ref TValue value, ref TOutput output, bool isConcurrent); + + PSFOperationStatus VisitSecondaryRead(ref TValue value, ref TInput input, ref TOutput output, long physicalAddress, bool tombstone, bool isConcurrent); + + PSFOperationStatus VisitSecondaryRead(ref TKey key, ref TValue value, ref TInput input, ref TOutput output, bool tombstone, bool isConcurrent); + #endregion Output visitors + } + + /// + /// The Functions for the TRecordId (which is the Value param to the secondary FasterKV); mostly pass-through + /// + /// The type of the user key in the primary Faster KV + /// The type of the user value in the primary Faster KV + internal class PSFPrimaryFunctions : StubbedFunctions>, + IPSFFunctions> + { + public long GroupId(ref PSFInputPrimaryReadAddress input) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); + + public bool IsDelete(ref PSFInputPrimaryReadAddress input) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); + public bool SetDelete(ref PSFInputPrimaryReadAddress input, bool value) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long ReadLogicalAddress(ref PSFInputPrimaryReadAddress input) => input.ReadLogicalAddress; + + public ref TKVKey QueryKeyRef(ref PSFInputPrimaryReadAddress input) => throw new PSFInternalErrorException("Cannot call this accessor on the primary FKV"); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PSFOperationStatus VisitPrimaryReadAddress(ref TKVKey key, ref TKVValue value, ref PSFOutputPrimaryReadAddress output, bool isConcurrent) + { + // Tombstone is not needed here; it is only needed for the chains in the secondary FKV. + output.ProviderDatas.Enqueue(new FasterKVProviderData(output.allocator, ref key, ref value)); + return new PSFOperationStatus(OperationStatus.SUCCESS); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PSFOperationStatus VisitSecondaryRead(ref TKVValue value, ref PSFInputPrimaryReadAddress input, ref PSFOutputPrimaryReadAddress output, + long physicalAddress, bool tombstone, bool isConcurrent) + => throw new PSFInternalErrorException("Cannot call this form of Visit() on the primary FKV"); // TODO review error messages + + public PSFOperationStatus VisitSecondaryRead(ref TKVKey key, ref TKVValue value, ref PSFInputPrimaryReadAddress input, ref PSFOutputPrimaryReadAddress output, + bool tombstone, bool isConcurrent) + => throw new PSFInternalErrorException("Cannot call this form of Visit() on the primary FKV"); // TODO review error messages + } + + /// + /// The Functions for the TRecordId (which is the Value param to the secondary FasterKV); mostly pass-through + /// + /// The type of the result key + /// The type of the value + public class PSFSecondaryFunctions : StubbedFunctions, PSFOutputSecondary>, + IPSFFunctions, PSFOutputSecondary> + where TPSFKey : struct + where TRecordId : struct + { + public long GroupId(ref PSFInputSecondary input) => input.GroupId; + + public bool IsDelete(ref PSFInputSecondary input) => input.IsDelete; + public bool SetDelete(ref PSFInputSecondary input, bool value) => input.IsDelete = value; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public long ReadLogicalAddress(ref PSFInputSecondary input) => input.ReadLogicalAddress; + + public ref TPSFKey QueryKeyRef(ref PSFInputSecondary input) => ref input.QueryKeyRef; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PSFOperationStatus VisitPrimaryReadAddress(ref TPSFKey key, ref TRecordId value, ref PSFOutputSecondary output, bool isConcurrent) + => throw new PSFInternalErrorException("Cannot call this form of Visit() on the secondary FKV"); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PSFOperationStatus VisitSecondaryRead(ref TRecordId value, ref PSFInputSecondary input, ref PSFOutputSecondary output, + long physicalAddress, bool tombstone, bool isConcurrent) + { + // This is the secondary FKV; we hold onto the RecordId and create the provider data when QueryPSF returns. + output.RecordId = value; + output.Tombstone = tombstone; + ref KeyPointer keyPointer = ref output.keyAccessor.GetKeyPointerRef(physicalAddress); + Debug.Assert(keyPointer.PsfOrdinal == input.PsfOrdinal, "Visit found mismatched PSF ordinal"); + output.PreviousLogicalAddress = keyPointer.PreviousAddress; + return new PSFOperationStatus(OperationStatus.SUCCESS); + } + + public PSFOperationStatus VisitSecondaryRead(ref TPSFKey key, ref TRecordId value, ref PSFInputSecondary input, ref PSFOutputSecondary output, + bool tombstone, bool isConcurrent) + { + // This is the secondary FKV; we hold onto the RecordId and create the provider data when QueryPSF returns. + output.RecordId = value; + output.Tombstone = tombstone; + ref CompositeKey compositeKey = ref CompositeKey.CastFromFirstKeyPointerRefAsKeyRef(ref key); + ref KeyPointer keyPointer = ref output.keyAccessor.GetKeyPointerRef(ref compositeKey, input.PsfOrdinal); + Debug.Assert(keyPointer.PsfOrdinal == input.PsfOrdinal, "Visit found mismatched PSF ordinal"); + output.PreviousLogicalAddress = keyPointer.PreviousAddress; + return new PSFOperationStatus(OperationStatus.SUCCESS); + } + } + + public class StubbedFunctions : IFunctions + { + // All IFunctions methods are unused; the IFunctions "implementation" is to satisfy the ClientSession requirement. We use only the Visit methods. + #region IFunctions stubs + private const string MustUseVisitMethod = "PSF-implementing FasterKVs must use one of the Visit* methods"; + + #region Upserts + public bool ConcurrentWriter(ref TKey _, ref TValue src, ref TValue dst) => throw new PSFInternalErrorException(MustUseVisitMethod); + + public void SingleWriter(ref TKey _, ref TValue src, ref TValue dst) => throw new PSFInternalErrorException(MustUseVisitMethod); + + public void UpsertCompletionCallback(ref TKey _, ref TValue value, PSFContext ctx) => throw new PSFInternalErrorException(MustUseVisitMethod); + #endregion Upserts + + #region Reads + public void ConcurrentReader(ref TKey key, ref TInput input, ref TValue value, ref TOutput dst) + => throw new PSFInternalErrorException(MustUseVisitMethod); + + public unsafe void SingleReader(ref TKey _, ref TInput input, ref TValue value, ref TOutput dst) + => throw new PSFInternalErrorException(MustUseVisitMethod); + + public void ReadCompletionCallback(ref TKey _, ref TInput input, ref TOutput output, PSFContext ctx, Status status) + => throw new PSFInternalErrorException(MustUseVisitMethod); + #endregion Reads + + #region RMWs + public void CopyUpdater(ref TKey _, ref TInput input, ref TValue oldValue, ref TValue newValue) + => throw new PSFInternalErrorException(MustUseVisitMethod); + + public void InitialUpdater(ref TKey _, ref TInput input, ref TValue value) + => throw new PSFInternalErrorException(MustUseVisitMethod); + + public bool InPlaceUpdater(ref TKey _, ref TInput input, ref TValue value) + => throw new PSFInternalErrorException(MustUseVisitMethod); + + public void RMWCompletionCallback(ref TKey _, ref TInput input, PSFContext ctx, Status status) + => throw new PSFInternalErrorException(MustUseVisitMethod); + #endregion RMWs + + public void DeleteCompletionCallback(ref TKey _, PSFContext ctx) + => throw new PSFInternalErrorException(MustUseVisitMethod); + + public void CheckpointCompletionCallback(string sessionId, CommitPoint commitPoint) + => throw new PSFInternalErrorException(MustUseVisitMethod); + #endregion IFunctions stubs + } +} diff --git a/cs/src/core/Index/PSF/PSFGroup.cs b/cs/src/core/Index/PSF/PSFGroup.cs new file mode 100644 index 000000000..d718b7a30 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFGroup.cs @@ -0,0 +1,447 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core.Index.PSF; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; +using System.Threading.Tasks; + +namespace FASTER.core +{ + /// + /// A group of s. Ideally, most records in the group will either match all + /// PSFs or none, for efficient use of log space. + /// + /// The type of the wrapper for the provider's data (obtained from TRecordId) + /// The type of the key returned by the Predicate and stored in the secondary FasterKV instance + /// The type of data record supplied by the data provider; in FasterKV it + /// is the logicalAddress of the record in the primary FasterKV instance. + public class PSFGroup : IExecutePSF, + IQueryPSF + where TPSFKey : struct + where TRecordId : struct + { + internal FasterKV fht; + internal IPSFDefinition[] psfDefinitions; + private readonly PSFRegistrationSettings regSettings; + + /// + /// ID of the group (used internally only) + /// + public long Id { get; } + + private readonly CheckpointSettings checkpointSettings; + private readonly int keyPointerSize = Utility.GetSize(default(KeyPointer)); + private readonly int recordIdSize = (Utility.GetSize(default(TRecordId)) + sizeof(long) - 1) & ~(sizeof(long) - 1); + + private SessionManager, PSFOutputSecondary, PSFSecondaryFunctions> SecondarySessions; + + /// + /// The list of s in this group + /// + public PSF[] PSFs { get; private set; } + + private int PSFCount => this.PSFs.Length; + + private readonly IFasterEqualityComparer userKeyComparer; + private readonly KeyAccessor keyAccessor; + + private readonly SectorAlignedBufferPool bufferPool; + + // Override equivalence testing for set membership + + /// + public override int GetHashCode() => this.Id.GetHashCode(); + + /// + public override bool Equals(object obj) => this.Equals(obj as PSFGroup); + + /// + public bool Equals(PSFGroup other) => !(other is null) && this.Id == other.Id; + + /// + /// Constructor + /// + /// Optional registration settings + /// PSF definitions + /// The ordinal of this PSFGroup in the 's + /// PSFGroup list. + public PSFGroup(PSFRegistrationSettings regSettings, IPSFDefinition[] defs, long id) + { + this.psfDefinitions = defs; + this.Id = id; + this.regSettings = regSettings; + this.userKeyComparer = GetUserKeyComparer(); + + this.PSFs = defs.Select((def, ord) => new PSF(this.Id, ord, def.Name, this)).ToArray(); + this.keyAccessor = new KeyAccessor(this.userKeyComparer, this.PSFCount, this.keyPointerSize); + + this.checkpointSettings = regSettings?.CheckpointSettings; + this.fht = new FasterKV( + regSettings.HashTableSize, regSettings.LogSettings, this.checkpointSettings, null /*SerializerSettings*/, + new CompositeKey.UnusedKeyComparer(), + new VariableLengthStructSettings + { + keyLength = new CompositeKey.VarLenLength(this.keyPointerSize, this.PSFCount) + } + ); + this.fht.hlog.PsfKeyAccessor = keyAccessor; + this.SecondarySessions = new SessionManager, PSFOutputSecondary, PSFSecondaryFunctions>( + this.fht, regSettings.ThreadAffinitized); + + this.bufferPool = this.fht.hlog.bufferPool; + } + + private IFasterEqualityComparer GetUserKeyComparer() + { + if (!(this.regSettings.KeyComparer is null)) + return this.regSettings.KeyComparer; + if (typeof(IFasterEqualityComparer).IsAssignableFrom(typeof(TPSFKey))) + return new TPSFKey() as IFasterEqualityComparer; + + Console.WriteLine( + $"***WARNING*** Creating default FASTER key equality comparer based on potentially slow {nameof(EqualityComparer)}." + + $" To avoid this, provide a comparer in {nameof(PSFRegistrationSettings)}.{nameof(PSFRegistrationSettings.KeyComparer)}," + + $" or make {typeof(TPSFKey).Name} implement the interface {nameof(IFasterEqualityComparer)}"); + return FasterEqualityComparer.Get(); + } + + /// + /// Returns the named from the PSFs list. + /// + /// The name of the ; unique among all groups + /// + public PSF this[string name] + => Array.Find(this.PSFs, psf => psf.Name.Equals(name, StringComparison.CurrentCultureIgnoreCase)) + ?? throw new PSFArgumentException("PSF not found"); + + private unsafe void StoreKeys(ref GroupCompositeKey keys, byte* keysPtr, int keysLen) + { + var poolKeyMem = this.bufferPool.Get(keysLen); + Buffer.MemoryCopy(keysPtr, poolKeyMem.GetValidPointer(), keysLen, keysLen); + keys.Set(poolKeyMem); + } + + internal unsafe void MarkChanges(ref GroupCompositeKeyPair keysPair) + { + ref GroupCompositeKey before = ref keysPair.Before; + ref GroupCompositeKey after = ref keysPair.After; + ref CompositeKey beforeCompositeKey = ref before.CastToKeyRef>(); + ref CompositeKey afterCompositeKey = ref after.CastToKeyRef>(); + for (var ii = 0; ii < this.PSFCount; ++ii) + { + ref KeyPointer beforeKeyPointer = ref beforeCompositeKey.GetKeyPointerRef(ii, this.keyPointerSize); + ref KeyPointer afterKeyPointer = ref afterCompositeKey.GetKeyPointerRef(ii, this.keyPointerSize); + + var beforeIsNull = beforeKeyPointer.IsNull; + var afterIsNull = afterKeyPointer.IsNull; + var keysEqual = !beforeIsNull && !afterIsNull && beforeKeyPointer.Key.Equals(afterKeyPointer.Key); + + // IsNull is already set in PSFGroup.ExecuteAndStore. + if ((beforeIsNull != afterIsNull) || !keysEqual) + keysPair.HasChanges = true; + if (!beforeIsNull && (afterIsNull || !keysEqual)) + { + afterKeyPointer.IsUnlinkOld = true; + keysPair.HasChanges = true; + } + if (!afterIsNull && (beforeIsNull || !keysEqual)) + { + afterKeyPointer.IsLinkNew = true; + keysPair.HasChanges = true; + } + } + } + + /// + public unsafe Status ExecuteAndStore(TProviderData providerData, TRecordId recordId, PSFExecutePhase phase, + PSFChangeTracker changeTracker) + { + // Note: stackalloc is safe because PendingContext or PSFChangeTracker will copy it to the bufferPool + // if needed. On the Insert fast path, we don't want any allocations otherwise; changeTracker is null. + var keyMemLen = this.keyPointerSize * this.PSFCount; + var keyBytes = stackalloc byte[keyMemLen]; + var anyMatch = false; + + for (var ii = 0; ii < this.PSFCount; ++ii) + { + ref KeyPointer keyPointer = ref Unsafe.AsRef>(keyBytes + ii * this.keyPointerSize); + keyPointer.PreviousAddress = Constants.kInvalidAddress; + keyPointer.PsfOrdinal = (byte)ii; + + var key = this.psfDefinitions[ii].Execute(providerData); + keyPointer.IsNull = !key.HasValue; + if (key.HasValue) + { + keyPointer.Key = key.Value; + anyMatch = true; + } + } + + if (!anyMatch && phase == PSFExecutePhase.Insert) + return Status.OK; + + ref CompositeKey compositeKey = ref Unsafe.AsRef>(keyBytes); + var input = new PSFInputSecondary(this.Id, 0); + var value = recordId; + + int groupOrdinal = -1; + if (!(changeTracker is null)) + { + value = changeTracker.BeforeRecordId; + if (phase == PSFExecutePhase.PreUpdate) + { + // Get a free group ref and store the "before" values. + ref GroupCompositeKeyPair groupKeysPair = ref changeTracker.FindGroupRef(this.Id); + StoreKeys(ref groupKeysPair.Before, keyBytes, keyMemLen); + return Status.OK; + } + + if (phase == PSFExecutePhase.PostUpdate) + { + // TODOtest: If not found, this is a new group added after the PreUpdate was done, so handle this as an insert. + if (!changeTracker.FindGroup(this.Id, out groupOrdinal)) + { + phase = PSFExecutePhase.Insert; + } + else + { + ref GroupCompositeKeyPair groupKeysPair = ref changeTracker.GetGroupRef(groupOrdinal); + StoreKeys(ref groupKeysPair.After, keyBytes, keyMemLen); + this.MarkChanges(ref groupKeysPair); + // TODOtest: In debug, for initial dev, follow chains to assert the values match what is in the record's compositeKey + if (!groupKeysPair.HasChanges) + return Status.OK; + } + } + + // We don't need to do anything here for Delete. + } + + var session = this.SecondarySessions.GetSession(); + try + { + var lsn = session.ctx.serialNum + 1; + var context = new PSFContext { Functions = session.functions }; + return phase switch + { + PSFExecutePhase.Insert => session.PsfInsert(ref compositeKey.CastToFirstKeyPointerRefAsKeyRef(), ref value, ref input, ref context, lsn), + PSFExecutePhase.PostUpdate => session.PsfUpdate(ref changeTracker.GetGroupRef(groupOrdinal), + ref value, ref input, ref context, lsn, changeTracker), + PSFExecutePhase.Delete => session.PsfDelete(ref compositeKey.CastToFirstKeyPointerRefAsKeyRef(), ref value, ref input, ref context, lsn, + changeTracker), + _ => throw new PSFInternalErrorException("Unknown PSF execution Phase {phase}") + }; + } + finally + { + this.SecondarySessions.ReleaseSession(session); + } + } + + /// + public Status GetBeforeKeys(PSFChangeTracker changeTracker) + { + if (changeTracker.HasBeforeKeys) + return Status.OK; + + // Obtain the "before" values. TODOcache: try to find TRecordId in the IPUCache first. + return ExecuteAndStore(changeTracker.BeforeData, default, PSFExecutePhase.PreUpdate, changeTracker); + } + + /// + /// Update the RecordId + /// + public Status Update(PSFChangeTracker changeTracker) + { + if (changeTracker.UpdateOp == UpdateOperation.Insert) + { + // RMW did not find the record so did an insert. Go through Insert logic here. + return this.ExecuteAndStore(changeTracker.BeforeData, changeTracker.BeforeRecordId, PSFExecutePhase.Insert, changeTracker:null); + } + + changeTracker.CachedBeforeLA = Constants.kInvalidAddress; // TODOcache: Find BeforeRecordId in IPUCache + if (changeTracker.CachedBeforeLA != Constants.kInvalidAddress) + { + if (changeTracker.UpdateOp == UpdateOperation.RCU) + { + // TODOcache: Tombstone it, and possibly unlink; or just copy its keys into changeTracker.Before. + } + else + { + // TODOcache: Try to splice in-place; or just copy its keys into changeTracker.Before. + } + } + else + { + if (this.GetBeforeKeys(changeTracker) != Status.OK) + { + // TODOerr: handle errors from GetBeforeKeys + } + } + return this.ExecuteAndStore(changeTracker.AfterData, default, PSFExecutePhase.PostUpdate, changeTracker); + } + + /// + /// Delete the RecordId + /// + public Status Delete(PSFChangeTracker changeTracker) + { + changeTracker.CachedBeforeLA = Constants.kInvalidAddress; // TODOcache: Find BeforeRecordId in IPUCache + if (changeTracker.CachedBeforeLA != Constants.kInvalidAddress) + { + // TODOcache: Tombstone it, and possibly unlink; or just copy its keys into changeTracker.Before. + // If the latter, we can bypass ExecuteAndStore's PSF-execute loop + } + return this.ExecuteAndStore(changeTracker.BeforeData, default, PSFExecutePhase.Delete, changeTracker); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private unsafe PSFInputSecondary MakeQueryInput(int psfOrdinal, ref TPSFKey key) + { + // Putting the query key in PSFInput is necessary because iterator functions cannot contain unsafe code or have + // byref args, and bufferPool is needed here because the stack goes away as part of the iterator operation. + var psfInput = new PSFInputSecondary(this.Id, psfOrdinal); + psfInput.SetQueryKey(this.bufferPool, this.keyAccessor, ref key); + return psfInput; + } + + /// + public unsafe IEnumerable Query(int psfOrdinal, TPSFKey key, PSFQuerySettings querySettings) + => Query(MakeQueryInput(psfOrdinal, ref key), querySettings); + + private IEnumerable Query(PSFInputSecondary input, PSFQuerySettings querySettings) + { + // TODOperf: if there are multiple PSFs within this group we can step through in parallel and return them + // as a single merged stream; will require multiple TPSFKeys and their indexes in queryKeyPtr. Also consider + // having TPSFKeys[] for a single PSF walk through in parallel, so the FHT log memory access is sequential. + var output = new PSFOutputSecondary(this.keyAccessor); + var session = this.SecondarySessions.GetSession(); + var context = new PSFContext { Functions = session.functions }; + var deadRecs = new DeadRecords(); + try + { + // Because we traverse the chain, we must wait for any pending read operations to complete. + // TODOperf: See if there is a better solution than spinWaiting in CompletePending. + Status status = session.PsfReadKey(ref input.QueryKeyRef, ref input, ref output, ref context, session.ctx.serialNum + 1); + if (querySettings.IsCanceled) + yield break; + if (status == Status.PENDING) + session.CompletePending(spinWait: true); + if (status != Status.OK) // TODOerr: check other status + yield break; + + if (output.Tombstone) + deadRecs.Add(output.RecordId); + else + yield return output.RecordId; + + do + { + input.ReadLogicalAddress = output.PreviousLogicalAddress; + status = session.PsfReadAddress(ref input, ref output, ref context, session.ctx.serialNum + 1); + if (status == Status.PENDING) + session.CompletePending(spinWait: true); + if (querySettings.IsCanceled) + yield break; + if (status != Status.OK) // TODOerr: check other status + yield break; + + if (deadRecs.IsDead(output.RecordId, output.Tombstone)) + continue; + + yield return output.RecordId; + } while (output.PreviousLogicalAddress != Constants.kInvalidAddress); + } + finally + { + this.SecondarySessions.ReleaseSession(session); + input.Dispose(); + } + } + +#if DOTNETCORE + /// + public unsafe IAsyncEnumerable QueryAsync(int psfOrdinal, TPSFKey key, PSFQuerySettings querySettings) + => QueryAsync(MakeQueryInput(psfOrdinal, ref key), querySettings); + + private async IAsyncEnumerable QueryAsync(PSFInputSecondary input, PSFQuerySettings querySettings) + { + // TODOperf: if there are multiple PSFs within this group we can step through in parallel and return them + // as a single merged stream; will require multiple TPSFKeys and their indexes in queryKeyPtr. Also consider + // having TPSFKeys[] for a single PSF walk through in parallel, so the FHT log memory access is sequential. + var output = new PSFOutputSecondary(this.keyAccessor); + var session = this.SecondarySessions.GetSession(); + var context = new PSFContext { Functions = session.functions }; + var deadRecs = new DeadRecords(); + try + { + // Because we traverse the chain, we must wait for any pending read operations to complete. + var readAsyncResult = await session.PsfReadKeyAsync(ref input.QueryKeyRef, ref input, ref output, ref context, session.ctx.serialNum + 1, querySettings); + if (querySettings.IsCanceled) + yield break; + var (status, _) = readAsyncResult.CompleteRead(); + if (status != Status.OK) // TODOerr: check other status + yield break; + + if (output.Tombstone) + deadRecs.Add(output.RecordId); + else + yield return output.RecordId; + + do + { + input.ReadLogicalAddress = output.PreviousLogicalAddress; + readAsyncResult = await session.PsfReadAddressAsync(ref input, ref output, ref context, session.ctx.serialNum + 1, querySettings); + if (querySettings.IsCanceled) + yield break; + (status, _) = readAsyncResult.CompleteRead(); + if (status != Status.OK) // TODOerr: check other status + yield break; + + if (deadRecs.IsDead(output.RecordId, output.Tombstone)) + continue; + + yield return output.RecordId; + } while (output.PreviousLogicalAddress != Constants.kInvalidAddress); + } + finally + { + this.SecondarySessions.ReleaseSession(session); + input.Dispose(); + } + } +#endif + + #region Checkpoint Operations + /// + public bool TakeFullCheckpoint() => this.fht.TakeFullCheckpoint(out _); + + /// + public ValueTask CompleteCheckpointAsync(CancellationToken token = default) => this.fht.CompleteCheckpointAsync(token); + + /// + public bool TakeIndexCheckpoint() => this.fht.TakeIndexCheckpoint(out _); + + /// + public bool TakeHybridLogCheckpoint() => this.fht.TakeHybridLogCheckpoint(out _); + + /// + public void Recover() => this.fht.Recover(); + #endregion Checkpoint Operations + + #region Log Operations + /// + public void FlushLog(bool wait) => this.fht.Log.Flush(wait); + + /// + public void FlushAndEvictLog(bool wait) => this.fht.Log.FlushAndEvict(wait); + + /// + public void DisposeLogFromMemory() => this.fht.Log.DisposeFromMemory(); + #endregion Log Operations + } +} diff --git a/cs/src/core/Index/PSF/PSFInput.cs b/cs/src/core/Index/PSF/PSFInput.cs new file mode 100644 index 000000000..5431ff6e0 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFInput.cs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Runtime.CompilerServices; + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +namespace FASTER.core +{ + /// + /// Input to PsfInternalReadAddress on the primary (stores user values) FasterKV to retrieve the Key and Value + /// for a logicalAddress returned from the secondary FasterKV instances. This class is FasterKV-provider-specific. + /// + public unsafe struct PSFInputPrimaryReadAddress + { + internal PSFInputPrimaryReadAddress(long readLA) + { + this.ReadLogicalAddress = readLA; + } + + public long ReadLogicalAddress { get; set; } + } + + /// + /// Input to operations on the secondary FasterKV instance (stores PSF chains) for everything + /// except reading based on a LogicalAddress. + /// + public unsafe struct PSFInputSecondary : IDisposable + where TPSFKey : new() + { + private SectorAlignedMemory keyPointerMem; + + internal PSFInputSecondary(long groupId, int psfOrdinal) + { + this.keyPointerMem = null; + this.GroupId = groupId; + this.PsfOrdinal = psfOrdinal; + this.IsDelete = false; + this.ReadLogicalAddress = Constants.kInvalidAddress; + } + + internal void SetQueryKey(SectorAlignedBufferPool pool, KeyAccessor keyAccessor, ref TPSFKey key) + { + // Create a varlen CompositeKey with just one item. This is ONLY used as the query key to QueryPSF. + this.keyPointerMem = pool.Get(keyAccessor.KeyPointerSize); + ref KeyPointer keyPointer = ref Unsafe.AsRef>(keyPointerMem.GetValidPointer()); + keyPointer.Initialize(this.PsfOrdinal, ref key); + } + + /// + /// The ID of the for this operation. + /// + public long GroupId { get; } + + /// + /// The ordinal of the in the group for this operation. + /// + public int PsfOrdinal { get; set; } + + /// + /// Whether this is a Delete (or the Delete part of an RCU) + /// + public bool IsDelete { get; set; } + + /// + /// The logical address to read in one of the PsfRead*Address methods + /// + public long ReadLogicalAddress { get; set; } + + /// + /// The query key for a QueryPSF method + /// + public ref TPSFKey QueryKeyRef => ref Unsafe.AsRef(this.keyPointerMem.GetValidPointer()); + + public void Dispose() + { + if (!(this.keyPointerMem is null)) + this.keyPointerMem.Return(); + } + } +} diff --git a/cs/src/core/Index/PSF/PSFManager.cs b/cs/src/core/Index/PSF/PSFManager.cs new file mode 100644 index 000000000..2ea2a33c5 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFManager.cs @@ -0,0 +1,905 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using FASTER.core.Index.PSF; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +// TODO: Remove PackageId and PackageOutputPath from csproj when this is folded into master +// TODO: Make a new FASTER.PSF.dll + +namespace FASTER.core +{ + /// + /// The class that manages PSFs. Called internally by the primary FasterKV. + /// + /// The type of the provider data returned by PSF queries; for the primary FasterKV, it is + /// The type of the Record identifier in the data provider; for the primary FasterKV it is the record's logical address + public class PSFManager where TRecordId : struct, IComparable + { + private readonly ConcurrentDictionary> psfGroups + = new ConcurrentDictionary>(); + + private readonly ConcurrentDictionary psfNames = new ConcurrentDictionary(); + + internal bool HasPSFs => this.psfGroups.Count > 0; + + /// + /// Inserts a new PSF key/RecordId, or adds the RecordId to an existing chain + /// + /// The provider's data; will be passed to the PSF execution + /// The record Id to be stored for any matching PSFs + /// Tracks changes if this is an existing Key/RecordId entry + /// A status code indicating the result of the operation + public Status Upsert(TProviderData data, TRecordId recordId, PSFChangeTracker changeTracker) + { + // TODO: RecordId locking, to ensure consistency of multiple PSFs if the same record is updated + // multiple times; possibly a single Array[N] which is locked on TRecordId.GetHashCode % N. + + // This Upsert was an Insert: For the FasterKV Insert fast path, changeTracker is null. + if (changeTracker is null || changeTracker.UpdateOp == UpdateOperation.Insert) + { + foreach (var group in this.psfGroups.Values) + { + // Fast Insert path: No IPUCache lookup is done for Inserts, so this is called directly here. + var status = group.ExecuteAndStore(data, recordId, PSFExecutePhase.Insert, changeTracker); + if (status != Status.OK) + { + // TODOerr: handle errors + } + } + return Status.OK; + } + + // This Upsert was an IPU or RCU + return this.Update(changeTracker); + } + + /// + /// Updates a PSF key/RecordId entry, possibly by RCU (Read-Copy-Update) + /// + /// Tracks changes for an existing Key/RecordId entry + /// A status code indicating the result of the operation + public Status Update(PSFChangeTracker changeTracker) + { + foreach (var group in this.psfGroups.Values) + { + var status = group.Update(changeTracker); + if (status != Status.OK) + { + // TODOerr: handle errors + } + } + return Status.OK; + } + + /// + /// Deletes a PSF key/RecordId entry from the chain, possibly by insertion of a "marked deleted" record + /// + /// Tracks changes for an existing Key/RecordId entry + /// A status code indicating the result of the operation + public Status Delete(PSFChangeTracker changeTracker) + { + foreach (var group in this.psfGroups.Values) + { + var status = group.Delete(changeTracker); + if (status != Status.OK) + { + // TODOerr: handle errors + } + } + return Status.OK; + } + + /// + /// Obtains a list of registered PSF names organized by the groups defined in previous RegisterPSF calls. + /// + /// A list of registered PSF names organized by the groups defined in previous RegisterPSF calls. + public string[][] GetRegisteredPSFNames() => throw new NotImplementedException("TODO"); + + /// + /// Creates an instance of a to track changes for an existing Key/RecordId entry. + /// + /// An instance of a to track changes for an existing Key/RecordId entry. + public PSFChangeTracker CreateChangeTracker() + => new PSFChangeTracker(this.psfGroups.Values.Select(group => group.Id)); + + /// + /// Sets the data for the state of a provider's data record prior to an update. + /// + /// Tracks changes for the Key/RecordId entry that will be updated. + /// The provider's data prior to the update; will be passed to the PSF execution + /// The record Id to be stored for any matching PSFs + /// Whether PSFs should be executed now or deferred. Should be 'true' if the provider's value type is an Object, + /// because the update will likely change the object's internal values, and thus a deferred 'before' execution will pick up the updated values instead. + /// A status code indicating the result of the operation + public Status SetBeforeData(PSFChangeTracker changeTracker, TProviderData data, TRecordId recordId, bool executePSFsNow) + { + changeTracker.SetBeforeData(data, recordId); + if (executePSFsNow) + { + foreach (var group in this.psfGroups.Values) + { + var status = group.GetBeforeKeys(changeTracker); + if (status != Status.OK) + { + // TODOerr: handle errors + } + } + changeTracker.HasBeforeKeys = true; + } + return Status.OK; + } + + /// + /// Sets the data for the state of a provider's data record after to an update. + /// + /// Tracks changes for the Key/RecordId entry that will be updated. + /// The provider's data after to the update; will be passed to the PSF execution + /// The record Id to be stored for any matching PSFs + /// A status code indicating the result of the operation + public Status SetAfterData(PSFChangeTracker changeTracker, TProviderData data, TRecordId recordId) + { + changeTracker.SetAfterData(data, recordId); + return Status.OK; + } + + private static long NextGroupId = 0; + + private void AddGroup(PSFGroup group) where TPSFKey : struct + { + var gId = Interlocked.Increment(ref NextGroupId); + this.psfGroups.TryAdd(gId - 1, group); + } + + private void VerifyIsBlittable() + { + if (!Utility.IsBlittable()) + throw new PSFArgumentException("The PSF Key type must be blittable."); + } + + private PSF GetImplementingPSF(IPSF ipsf) + { + if (ipsf is null) + throw new PSFArgumentException($"The PSF cannot be null."); + var psf = ipsf as PSF; + Guid id = default; + if (psf is null || !this.psfNames.TryGetValue(psf.Name, out id) || id != psf.Id) + throw new PSFArgumentException($"The PSF {psf.Name} with Id {(psf is null ? "(unavailable)" : id.ToString())} is not registered with this FasterKV."); + return psf; + } + + private void VerifyIsOurPSF(params IPSF[] psfs) + { + foreach (var psf in psfs) + { + if (psf is null) + throw new PSFArgumentException($"The PSF cannot be null."); + if (!this.psfNames.ContainsKey(psf.Name)) + throw new PSFArgumentException($"The PSF {psf.Name} is not registered with this FasterKV."); + } + } + + private void VerifyIsOurPSF(IEnumerable<(IPSF, IEnumerable)> psfsAndKeys) + { + if (psfsAndKeys is null) + throw new PSFArgumentException($"The PSF enumerable cannot be null."); + foreach (var psfAndKeys in psfsAndKeys) + this.VerifyIsOurPSF(psfAndKeys.Item1); + } + + private void VerifyIsOurPSF(IEnumerable<(IPSF, IEnumerable)> psfsAndKeys1, + IEnumerable<(IPSF, IEnumerable)> psfsAndKeys2) + { + VerifyIsOurPSF(psfsAndKeys1); + VerifyIsOurPSF(psfsAndKeys2); + } + + private void VerifyIsOurPSF(IEnumerable<(IPSF, IEnumerable)> psfsAndKeys1, + IEnumerable<(IPSF, IEnumerable)> psfsAndKeys2, + IEnumerable<(IPSF, IEnumerable)> psfsAndKeys3) + { + VerifyIsOurPSF(psfsAndKeys1); + VerifyIsOurPSF(psfsAndKeys2); + VerifyIsOurPSF(psfsAndKeys3); + } + + private static void VerifyRegistrationSettings(PSFRegistrationSettings registrationSettings) where TPSFKey : struct + { + if (registrationSettings is null) + throw new PSFArgumentException("PSFRegistrationSettings is required"); + if (registrationSettings.LogSettings is null) + throw new PSFArgumentException("PSFRegistrationSettings.LogSettings is required"); + if (registrationSettings.CheckpointSettings is null) + throw new PSFArgumentException("PSFRegistrationSettings.CheckpointSettings is required"); + + // TODOdcr: Support ReadCache and CopyReadsToTail for PSFs + if (!(registrationSettings.LogSettings.ReadCacheSettings is null) || registrationSettings.LogSettings.CopyReadsToTail) + throw new PSFArgumentException("PSFs do not support ReadCache or CopyReadsToTail"); + } + + /// + /// Register a with a simple definition. + /// + /// The type of the key returned from the + /// Registration settings for the secondary FasterKV instances, etc. + /// The PSF definition + /// A PSF implementation( + public IPSF RegisterPSF(PSFRegistrationSettings registrationSettings, IPSFDefinition def) + where TPSFKey : struct + { + this.VerifyIsBlittable(); + VerifyRegistrationSettings(registrationSettings); + if (def is null) + throw new PSFArgumentException("PSF definition cannot be null"); + + // This is a very rare operation and unlikely to have any contention, and locking the dictionary + // makes it much easier to recover from duplicates if needed. + lock (this.psfNames) + { + if (psfNames.ContainsKey(def.Name)) + throw new PSFArgumentException($"A PSF named {def.Name} is already registered in another group"); + var group = new PSFGroup(registrationSettings, new[] { def }, this.psfGroups.Count); + AddGroup(group); + var psf = group[def.Name]; + this.psfNames.TryAdd(psf.Name, psf.Id); + return psf; + } + } + + /// + /// Register multiple s with a vector of definitions. + /// + /// The type of the key returned from the + /// Registration settings for the secondary FasterKV instances, etc. + /// The PSF definitions + /// A PSF implementation( + public IPSF[] RegisterPSF(PSFRegistrationSettings registrationSettings, IPSFDefinition[] defs) + where TPSFKey : struct + { + this.VerifyIsBlittable(); + VerifyRegistrationSettings(registrationSettings); + if (defs is null || defs.Length == 0 || defs.Any(def => def is null) || defs.Length == 0) + throw new PSFArgumentException("PSF definitions cannot be null or empty"); + + // We use stackalloc for speed and can recurse in pending operations, so make sure we don't blow the stack. + if (defs.Length > Constants.kInvalidPsfOrdinal) + throw new PSFArgumentException($"There can be no more than {Constants.kInvalidPsfOrdinal} PSFs in a single Group"); + const int maxKeySize = 256; + if (Utility.GetSize(default(KeyPointer)) > maxKeySize) + throw new PSFArgumentException($"The size of the PSF key can be no more than {maxKeySize} bytes"); + + // This is a very rare operation and unlikely to have any contention, and locking the dictionary + // makes it much easier to recover from duplicates if needed. + lock (this.psfNames) + { + for (var ii = 0; ii < defs.Length; ++ii) + { + var def = defs[ii]; + if (psfNames.ContainsKey(def.Name)) + throw new PSFArgumentException($"A PSF named {def.Name} is already registered in another group"); + for (var jj = ii + 1; jj < defs.Length; ++jj) + { + if (defs[jj].Name == def.Name) + throw new PSFArgumentException($"The PSF name {def.Name} cannot be specfied twice"); + } + } + + var group = new PSFGroup(registrationSettings, defs, this.psfGroups.Count); + AddGroup(group); + foreach (var psf in group.PSFs) + this.psfNames.TryAdd(psf.Name, psf.Id); + return group.PSFs; + } + } + + /// + /// Does a synchronous scan of a single PSF for records matching a single key + /// + /// The type of the key returned from the + /// The PSF to be queried + /// The identifying the records to be retrieved + /// Options for the PSF query operation + /// An enumeration of the s matching + public IEnumerable QueryPSF(IPSF psf, TPSFKey key, PSFQuerySettings querySettings) + where TPSFKey : struct + { + var psfImpl = this.GetImplementingPSF(psf); + querySettings ??= PSFQuerySettings.Default; + foreach (var recordId in psfImpl.Query(key, querySettings)) + { + if (querySettings.IsCanceled) + yield break; + yield return recordId; + } + } + +#if DOTNETCORE + /// + /// Does an asynchronous scan of a single PSF for records matching a single key + /// + /// The type of the key returned from the + /// The PSF to be queried + /// The identifying the records to be retrieved + /// Options for the PSF query operation + /// An async enumeration of the s matching + public async IAsyncEnumerable QueryPSFAsync(IPSF psf, TPSFKey key, PSFQuerySettings querySettings) + where TPSFKey : struct + { + var psfImpl = this.GetImplementingPSF(psf); + querySettings ??= PSFQuerySettings.Default; + await foreach (var recordId in psfImpl.QueryAsync(key, querySettings)) + { + if (querySettings.IsCanceled) + yield break; + yield return recordId; + } + } + +#endif // DOTNETCORE + + /// + /// Does a synchronous scan of a single PSF for records matching any of multiple keys, unioning the results. + /// + /// The type of the key returned from the + /// The PSF to be queried + /// The s identifying the records to be retrieved + /// Options for the PSF query operation + /// An enumeration of the s matching + public IEnumerable QueryPSF(IPSF psf, IEnumerable keys, PSFQuerySettings querySettings) + where TPSFKey : struct + { + this.VerifyIsOurPSF(psf); + querySettings ??= PSFQuerySettings.Default; + + // The recordIds cannot overlap between keys (unless something's gone wrong), so return them all. + // TODOperf: Consider a PQ ordered on secondary FKV LA so we can walk through in parallel (and in memory sequence) in one PsfRead(Key|Address) loop. + foreach (var key in keys) + { + foreach (var recordId in QueryPSF(psf, key, querySettings)) + { + if (querySettings.IsCanceled) + yield break; + yield return recordId; + } + } + } + +#if DOTNETCORE + /// + /// Does an asynchronous scan of a single PSF for records matching any of multiple keys, unioning the results. + /// + /// The type of the key returned from the + /// The PSF to be queried + /// The s identifying the records to be retrieved + /// Options for the PSF query operation + /// An async enumeration of the s matching + public async IAsyncEnumerable QueryPSFAsync(IPSF psf, IEnumerable keys, PSFQuerySettings querySettings) + where TPSFKey : struct + { + this.VerifyIsOurPSF(psf); + querySettings ??= PSFQuerySettings.Default; + + // The recordIds cannot overlap between keys (unless something's gone wrong), so return them all. + // TODOperf: Consider a PQ ordered on secondary FKV LA so we can walk through in parallel (and in memory sequence) in one PsfRead(Key|Address) loop. + foreach (var key in keys) + { + await foreach (var recordId in QueryPSFAsync(psf, key, querySettings)) + { + if (querySettings.IsCanceled) + yield break; + yield return recordId; + } + } + } + +#endif // DOTNETCORE + + /// + /// Does a synchronous scan of one key on each of two PSFs, returning records matching these keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The first PSF to be queried + /// The second PSF to be queried + /// The identifying the records to be retrieved from + /// The identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and + public IEnumerable QueryPSF( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + Func matchPredicate, + PSFQuerySettings querySettings) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + this.VerifyIsOurPSF(psf1, psf2); + querySettings ??= PSFQuerySettings.Default; + + return new QueryRecordIterator(psf1, this.QueryPSF(psf1, key1, querySettings), psf2, this.QueryPSF(psf2, key2, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0]), querySettings).Run(); + } + +#if DOTNETCORE + /// + /// Does an synchronous scan of one key on each of two PSFs, returning records matching these keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The first PSF to be queried + /// The second PSF to be queried + /// The identifying the records to be retrieved from + /// The identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and + public IAsyncEnumerable QueryPSFAsync( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + Func matchPredicate, + PSFQuerySettings querySettings) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + this.VerifyIsOurPSF(psf1, psf2); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(psf1, this.QueryPSFAsync(psf1, key1, querySettings), psf2, this.QueryPSFAsync(psf2, key2, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0]), querySettings).Run(); + } + +#endif // DOTNETCORE + + /// + /// Does a synchronous scan of multiple keys on each of two PSFs, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The first PSF to be queried + /// The second PSF to be queried + /// The s identifying the records to be retrieved from + /// The s identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and + public IEnumerable QueryPSF( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + Func matchPredicate, + PSFQuerySettings querySettings) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + this.VerifyIsOurPSF(psf1, psf2); + querySettings ??= PSFQuerySettings.Default; + + return new QueryRecordIterator(psf1, this.QueryPSF(psf1, keys1, querySettings), psf2, this.QueryPSF(psf2, keys2, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0]), querySettings).Run(); + } + +#if DOTNETCORE + /// + /// Does an asynchronous scan of multiple keys on each of two PSFs, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The first PSF to be queried + /// The second PSF to be queried + /// The s identifying the records to be retrieved from + /// The s identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and + public IAsyncEnumerable QueryPSFAsync( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + Func matchPredicate, + PSFQuerySettings querySettings) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + this.VerifyIsOurPSF(psf1, psf2); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(psf1, this.QueryPSFAsync(psf1, keys1, querySettings), psf2, this.QueryPSFAsync(psf2, keys2, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0]), querySettings).Run(); + } +#endif // DOTNETCORE + + /// + /// Does a synchronous scan of one key on each of three PSFs, returning records matching these keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The type of the key returned from the third + /// The first PSF to be queried + /// The second PSF to be queried + /// The third PSF to be queried + /// The identifying the records to be retrieved from + /// The identifying the records to be retrieved from + /// The identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and + public IEnumerable QueryPSF( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + IPSF psf3, TPSFKey3 key3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + this.VerifyIsOurPSF(psf1, psf2, psf3); + querySettings ??= PSFQuerySettings.Default; + + return new QueryRecordIterator(psf1, this.QueryPSF(psf1, key1, querySettings), psf2, this.QueryPSF(psf2, key2, querySettings), + psf3, this.QueryPSF(psf3, key3, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0], matchIndicators[2][0]), querySettings).Run(); + } + +#if DOTNETCORE + /// + /// Does an asynchronous scan of one key on each of three PSFs, returning records matching these keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The type of the key returned from the third + /// The first PSF to be queried + /// The second PSF to be queried + /// The third PSF to be queried + /// The identifying the records to be retrieved from + /// The identifying the records to be retrieved from + /// The identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and + public IAsyncEnumerable QueryPSFAsync( + IPSF psf1, TPSFKey1 key1, + IPSF psf2, TPSFKey2 key2, + IPSF psf3, TPSFKey3 key3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + this.VerifyIsOurPSF(psf1, psf2, psf3); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(psf1, this.QueryPSFAsync(psf1, key1, querySettings), psf2, this.QueryPSFAsync(psf2, key2, querySettings), + psf3, this.QueryPSFAsync(psf3, key3, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0], matchIndicators[2][0]), querySettings).Run(); + } +#endif // DOTNETCORE + + /// + /// Does a synchronous scan of multiple keys on each of three PSFs, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The type of the key returned from the third + /// The first PSF to be queried + /// The second PSF to be queried + /// The third PSF to be queried + /// The s identifying the records to be retrieved from + /// The s identifying the records to be retrieved from + /// The s identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and + public IEnumerable QueryPSF( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + IPSF psf3, IEnumerable keys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + this.VerifyIsOurPSF(psf1, psf2, psf3); + querySettings ??= PSFQuerySettings.Default; + + return new QueryRecordIterator(psf1, this.QueryPSF(psf1, keys1, querySettings), psf2, this.QueryPSF(psf2, keys2, querySettings), + psf3, this.QueryPSF(psf3, keys3, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0], matchIndicators[2][0]), querySettings).Run(); + } + +#if DOTNETCORE + /// + /// Does an asynchronous scan of multiple keys on each of three PSFs, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first + /// The type of the key returned from the second + /// The type of the key returned from the third + /// The first PSF to be queried + /// The second PSF to be queried + /// The third PSF to be queried + /// The s identifying the records to be retrieved from + /// The s identifying the records to be retrieved from + /// The s identifying the records to be retrieved from + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and + public IAsyncEnumerable QueryPSFAsync( + IPSF psf1, IEnumerable keys1, + IPSF psf2, IEnumerable keys2, + IPSF psf3, IEnumerable keys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + this.VerifyIsOurPSF(psf1, psf2, psf3); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(psf1, this.QueryPSFAsync(psf1, keys1, querySettings), psf2, this.QueryPSFAsync(psf2, keys2, querySettings), + psf3, this.QueryPSFAsync(psf3, keys3, querySettings), + matchIndicators => matchPredicate(matchIndicators[0][0], matchIndicators[1][0], matchIndicators[2][0]), querySettings).Run(); + } +#endif // DOTNETCORE + + // Power user versions. Anything more complicated than these can be post-processed with LINQ. + + /// + /// Does a synchronous scan of multiple keys on each of multiple PSFs with the same type, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the s + /// An enumeration of tuples containing a and the s to be queried on it + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and + public IEnumerable QueryPSF( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + this.VerifyIsOurPSF(psfsAndKeys); + querySettings ??= PSFQuerySettings.Default; + + return new QueryRecordIterator(new[] { psfsAndKeys.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings))) }, + matchIndicators => matchPredicate(matchIndicators[0]), querySettings).Run(); + } + +#if DOTNETCORE + /// + /// Does an asynchronous scan of multiple keys on each of multiple PSFs with the same type, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the s + /// An enumeration of tuples containing a and the s to be queried on it + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and + public IAsyncEnumerable QueryPSFAsync( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey : struct + { + this.VerifyIsOurPSF(psfsAndKeys); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(new[] { psfsAndKeys.Select(tup => ((IPSF)tup.psf, this.QueryPSFAsync(tup.psf, tup.keys, querySettings))) }, + matchIndicators => matchPredicate(matchIndicators[0]), querySettings).Run(); + } +#endif // DOTNETCORE + + /// + /// Does a synchronous scan of multiple keys on each of multiple PSFs on each of two TPSFKey types, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first set of s + /// The type of the key returned from the second set of s + /// The first enumeration of tuples containing a and the TPSFKey to be queried on it + /// The second enumeration of tuples containing a and the TPSFKey to be queried on it + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and + public IEnumerable QueryPSF( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + this.VerifyIsOurPSF(psfsAndKeys1, psfsAndKeys2); + querySettings ??= PSFQuerySettings.Default; + + return new QueryRecordIterator(new[] {psfsAndKeys1.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings))), + psfsAndKeys2.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings)))}, + matchIndicators => matchPredicate(matchIndicators[0], matchIndicators[1]), querySettings).Run(); + } + +#if DOTNETCORE + /// + /// Does an asynchronous scan of multiple keys on each of multiple PSFs on each of two TPSFKey types, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first set of s + /// The type of the key returned from the second set of s + /// The first enumeration of tuples containing a and the TPSFKey to be queried on it + /// The second enumeration of tuples containing a and the TPSFKey to be queried on it + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and + public IAsyncEnumerable QueryPSFAsync( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + { + this.VerifyIsOurPSF(psfsAndKeys1, psfsAndKeys2); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(new[] {psfsAndKeys1.Select(tup => ((IPSF)tup.psf, this.QueryPSFAsync(tup.psf, tup.keys, querySettings))), + psfsAndKeys2.Select(tup => ((IPSF)tup.psf, this.QueryPSFAsync(tup.psf, tup.keys, querySettings)))}, + matchIndicators => matchPredicate(matchIndicators[0], matchIndicators[1]), querySettings).Run(); + } +#endif // DOTNETCORE + + /// + /// Does a synchronous scan of multiple keys on each of multiple PSFs on each of three TPSFKey types, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first set of s + /// The type of the key returned from the second set of s + /// The type of the key returned from the third set of s + /// The first enumeration of tuples containing a and the TPSFKey to be queried on it + /// The second enumeration of tuples containing a and the TPSFKey to be queried on it + /// The third enumeration of tuples containing a and the TPSFKey to be queried on it + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An enumeration of the s matching the PSF keys and + public IEnumerable QueryPSF( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + this.VerifyIsOurPSF(psfsAndKeys1, psfsAndKeys2, psfsAndKeys3); + querySettings ??= PSFQuerySettings.Default; + + return new QueryRecordIterator(new[] {psfsAndKeys1.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings))), + psfsAndKeys2.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings))), + psfsAndKeys3.Select(tup => ((IPSF)tup.psf, this.QueryPSF(tup.psf, tup.keys, querySettings)))}, + matchIndicators => matchPredicate(matchIndicators[0], matchIndicators[1], matchIndicators[2]), querySettings).Run(); + } + +#if DOTNETCORE + /// + /// Does an asynchronous scan of multiple keys on each of multiple PSFs on each of three TPSFKey types, returning records matching any of those keys, with a union or intersection defined by + /// + /// The type of the key returned from the first set of s + /// The type of the key returned from the second set of s + /// The type of the key returned from the third set of s + /// The first enumeration of tuples containing a and the TPSFKey to be queried on it + /// The second enumeration of tuples containing a and the TPSFKey to be queried on it + /// The third enumeration of tuples containing a and the TPSFKey to be queried on it + /// Takes boolean parameters indicating which PSFs are matched by the current record, and returns a boolean indicating whether + /// that record should be included in the result set + /// Options for the PSF query operation + /// An async enumeration of the s matching the PSF keys and + public IAsyncEnumerable QueryPSFAsync( + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys1, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys2, + IEnumerable<(IPSF psf, IEnumerable keys)> psfsAndKeys3, + Func matchPredicate, + PSFQuerySettings querySettings = null) + where TPSFKey1 : struct + where TPSFKey2 : struct + where TPSFKey3 : struct + { + this.VerifyIsOurPSF(psfsAndKeys1, psfsAndKeys2, psfsAndKeys3); + querySettings ??= PSFQuerySettings.Default; + + return new AsyncQueryRecordIterator(new[] {psfsAndKeys1.Select(tup => ((IPSF)tup.psf, this.QueryPSFAsync(tup.psf, tup.keys, querySettings))), + psfsAndKeys2.Select(tup => ((IPSF)tup.psf, this.QueryPSFAsync(tup.psf, tup.keys, querySettings))), + psfsAndKeys3.Select(tup => ((IPSF)tup.psf, this.QueryPSFAsync(tup.psf, tup.keys, querySettings)))}, + matchIndicators => matchPredicate(matchIndicators[0], matchIndicators[1], matchIndicators[2]), querySettings).Run(); + } +#endif // DOTNETCORE + + #region Checkpoint Operations + // TODO Separate Tasks for each group's commit/restore operations? + + /// + /// For each , take a full checkpoint of the FasterKV implementing the group's PSFs. + /// + public bool TakeFullCheckpoint() + => this.psfGroups.Values.Aggregate(true, (result, group) => group.TakeFullCheckpoint() && result); + + /// + /// For each , complete ongoing checkpoint (spin-wait) + /// + public Task CompleteCheckpointAsync(CancellationToken token = default) + { + var tasks = this.psfGroups.Values.Select(group => group.CompleteCheckpointAsync(token).AsTask()).ToArray(); + return Task.WhenAll(tasks); + } + + /// + /// For each , take a checkpoint of the Index (hashtable) only + /// + public bool TakeIndexCheckpoint() + => this.psfGroups.Values.Aggregate(true, (result, group) => group.TakeIndexCheckpoint() && result); + + /// + /// For each , take a checkpoint of the hybrid log only + /// + public bool TakeHybridLogCheckpoint() + => this.psfGroups.Values.Aggregate(true, (result, group) => group.TakeHybridLogCheckpoint() && result); + + /// + /// For each , recover from last successful checkpoints + /// + public void Recover() + { + foreach (var group in this.psfGroups.Values) + group.Recover(); + } + #endregion Checkpoint Operations + + #region Log Operations + + /// + /// Flush logs for all s until their current tail (records are still retained in memory) + /// + /// Synchronous wait for operation to complete + public void FlushLogs(bool wait) + { + foreach (var group in this.psfGroups.Values) + group.FlushLog(wait); + } + + /// + /// Flush logs for all s and evict all records from memory + /// + /// Synchronous wait for operation to complete + /// When wait is false, this tells whether the full eviction was successfully registered with FASTER + public void FlushAndEvictLogs(bool wait) + { + foreach (var group in this.psfGroups.Values) + { + group.FlushAndEvictLog(wait); + } + } + + /// + /// Delete logs for all s entirely from memory. Cannot allocate on the log + /// after this point. This is a synchronous operation. + /// + public void DisposeLogsFromMemory() + { + foreach (var group in this.psfGroups.Values) + group.DisposeLogFromMemory(); + } + #endregion Log Operations + } +} diff --git a/cs/src/core/Index/PSF/PSFOperationStatus.cs b/cs/src/core/Index/PSF/PSFOperationStatus.cs new file mode 100644 index 000000000..835cdb085 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFOperationStatus.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// Wrapper for the non-public OperationStatus + /// + public struct PSFOperationStatus + { + internal OperationStatus Status; + + internal PSFOperationStatus(OperationStatus opStatus) => this.Status = opStatus; + } +} diff --git a/cs/src/core/Index/PSF/PSFOutput.cs b/cs/src/core/Index/PSF/PSFOutput.cs new file mode 100644 index 000000000..516ed366a --- /dev/null +++ b/cs/src/core/Index/PSF/PSFOutput.cs @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + /// + /// Output from ReadInternal on the primary (stores user values) FasterKV instance when reading + /// based on a LogicalAddress rather than a key. This class is FasterKV-provider-specific. + /// + /// The type of the key for user values + /// The type of the user values + public unsafe struct PSFOutputPrimaryReadAddress + { + internal readonly AllocatorBase allocator; + + internal ConcurrentQueue> ProviderDatas { get; private set; } + + internal PSFOutputPrimaryReadAddress(AllocatorBase alloc, + ConcurrentQueue> provDatas) + { + this.allocator = alloc; + this.ProviderDatas = provDatas; + } +#if false + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PSFOperationStatus Visit(int psfOrdinal, ref TKVKey key, ref TKVValue value, bool tombstone, bool isConcurrent) + { + // tombstone is not used here; it is only needed for the chains in the secondary FKV. + this.ProviderDatas.Enqueue(new FasterKVProviderData(this.allocator, ref key, ref value)); + return new PSFOperationStatus(OperationStatus.SUCCESS); + } + + public PSFOperationStatus Visit(int psfOrdinal, long physicalAddress, ref TKVValue value, bool tombstone, bool isConcurrent) + => throw new PSFInternalErrorException("Cannot call this form of Visit() on the primary FKV"); // TODO review error messages +#endif + } + + /// + /// Output from operations on the secondary FasterKV instance (stores PSF chains). + /// + /// The type of the key returned from a + /// The type of the provider's record identifier + public unsafe struct PSFOutputSecondary + where TPSFKey : new() + where TRecordId : new() + { + internal readonly KeyAccessor keyAccessor; + + internal TRecordId RecordId { get; set; } + + internal bool Tombstone { get; set; } + + internal long PreviousLogicalAddress { get; set; } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal PSFOutputSecondary(KeyAccessor keyAcc) + { + this.keyAccessor = keyAcc; + this.RecordId = default; + this.Tombstone = false; + this.PreviousLogicalAddress = Constants.kInvalidAddress; + } + +#if false + public PSFOperationStatus Visit(int psfOrdinal, ref TPSFKey key, + ref TRecordId value, bool tombstone, bool isConcurrent) + { + // This is the secondary FKV; we hold onto the RecordId and create the provider data when QueryPSF returns. + this.RecordId = value; + this.Tombstone = tombstone; + ref CompositeKey compositeKey = ref CompositeKey.CastFromFirstKeyPointerRefAsKeyRef(ref key); + ref KeyPointer keyPointer = ref this.keyAccessor.GetKeyPointerRef(ref compositeKey, psfOrdinal); + Debug.Assert(keyPointer.PsfOrdinal == (ushort)psfOrdinal, "Visit found mismatched PSF ordinal"); + this.PreviousLogicalAddress = keyPointer.PreviousAddress; + return new PSFOperationStatus(OperationStatus.SUCCESS); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public PSFOperationStatus Visit(int psfOrdinal, long physicalAddress, + ref TRecordId value, bool tombstone, bool isConcurrent) + { + // This is the secondary FKV; we hold onto the RecordId and create the provider data when QueryPSF returns. + this.RecordId = value; + this.Tombstone = tombstone; + ref KeyPointer keyPointer = ref this.keyAccessor.GetKeyPointerRef(physicalAddress); + Debug.Assert(keyPointer.PsfOrdinal == (ushort)psfOrdinal, "Visit found mismatched PSF ordinal"); + this.PreviousLogicalAddress = keyPointer.PreviousAddress; + return new PSFOperationStatus(OperationStatus.SUCCESS); + } +#endif + } +} diff --git a/cs/src/core/Index/PSF/PSFQuerySettings.cs b/cs/src/core/Index/PSF/PSFQuerySettings.cs new file mode 100644 index 000000000..f45ca64d1 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFQuerySettings.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Threading; + +namespace FASTER.core +{ + /// + /// Defines settings that control some behaviors of the QueryPSF execution. + /// + public class PSFQuerySettings + { + /// One or more streams has ended. Inputs are the PSF whose stream ended and the index of that PSF in the parameters, + /// identified by the 0-based ordinal of the TPSFKey type (TPSFKey1, TPSFKey2, or TPSFkey3 in the QueryPSF overloads) and + /// the 0-based ordinal of the PSF within the TPSFKey type. + /// true to continue the enumeration, else false + public Func OnStreamEnded; + + /// Cancel the enumeration if set. Can be set by another thread, + /// e.g. one presenting results to a UI, or by StreamEnded. + /// + public CancellationToken CancellationToken { get; set; } + + /// When cancellation is reqested, simply terminate the enumeration + /// without throwing a CancellationException. + public bool ThrowOnCancellation { get; set; } + + internal bool IsCanceled + { + get + { + if (this.CancellationToken.IsCancellationRequested) + { + if (this.ThrowOnCancellation) + CancellationToken.ThrowIfCancellationRequested(); + return true; + } + return false; + } + } + + internal bool CancelOnEOS(IPSF psf, (int, int) location) => !(this.OnStreamEnded is null) && !this.OnStreamEnded(psf, location); + + // Default is to let all streams continue to completion. + internal static readonly PSFQuerySettings Default = new PSFQuerySettings { OnStreamEnded = (unusedPsf, unusedIndex) => true }; + } +} diff --git a/cs/src/core/Index/PSF/PSFReadArgs.cs b/cs/src/core/Index/PSF/PSFReadArgs.cs new file mode 100644 index 000000000..755d5f607 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFReadArgs.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + // TODO: Move out of PSFs and include the new chained-LA Read function. + internal struct PSFReadArgs + { + internal readonly long LivenessCheckLogicalAddress; + + internal PSFReadArgs(long livenessCheckAddress) + { + this.LivenessCheckLogicalAddress = livenessCheckAddress; + } + } +} diff --git a/cs/src/core/Index/PSF/PSFRegistrationSettings.cs b/cs/src/core/Index/PSF/PSFRegistrationSettings.cs new file mode 100644 index 000000000..562100a72 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFRegistrationSettings.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// Options for PSF registration. + /// + public class PSFRegistrationSettings + { + /// + /// When registring new PSFs over an existing store, this is the logicalAddress in the primary + /// FasterKV at which indexing will be started. TODO: LogicalAddress is FasterKV-specific; revisit when indexing existing records. + /// + public long IndexFromAddress = Constants.kInvalidAddress; + + /// + /// The hash table size to be used in the PSF-implementing secondary FasterKV instances. + /// For PSFs defined on a FasterKV instance, if this is 0 or less, it will use the same value + /// passed to the primary FasterKV instance. + /// + public long HashTableSize = 0; + + /// + /// The log settings to be used in the PSF-implementing secondary FasterKV instances. + /// For PSFs defined on a FasterKV instance, if this is null, it will use the same settings as + /// passed to the primary FasterKV instance. + /// + public LogSettings LogSettings; + + /// + /// The log settings to be used in the PSF-implementing secondary FasterKV instances. + /// For PSFs defined on a FasterKV instance, if this is null, it will use the same settings + /// consistent with those passed to the primary FasterKV instance. + /// + public CheckpointSettings CheckpointSettings; + + /// + /// Optional key comparer; if null, should implement + /// ; otherwise a slower EqualityComparer will be used. + /// + public IFasterEqualityComparer KeyComparer; + + /// + /// Indicates whether PSFGroup Sessions are thread-affinitized. + /// + public bool ThreadAffinitized; + + /// + /// The size of the first IPU Cache; inserts are done into this cache only. If zero, no caching is done. // TODOCache + /// + public long IPU1CacheSize = 0; + + /// + /// The size of the second IPU Cache; inserts are not done into this cache, so more distant records + /// are likelier to remain. If this is nonzero, must also be nonzero. // TODOCache + /// + public long IPU2CacheSize = 0; + } +} diff --git a/cs/src/core/Index/PSF/PSFUpdateArgs.cs b/cs/src/core/Index/PSF/PSFUpdateArgs.cs new file mode 100644 index 000000000..974ae78f4 --- /dev/null +++ b/cs/src/core/Index/PSF/PSFUpdateArgs.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +namespace FASTER.core +{ + /// + /// Structure passed to and set by Upsert and RMW in the Primary FKV; specific to the FasterKV client. + /// + internal struct PSFUpdateArgs + { + /// + /// Set to the inserted or updated logical address, which is the RecordId for the FasterKV client + /// + /// Having this outside the changeTracker means that a non-updating Upsert does not incur + /// allocation overhead. + internal long LogicalAddress; + + /// + /// Created and populated with PreUpdate values on RMW or Upsert of an existing key + /// + internal PSFChangeTracker, long> ChangeTracker; + } +} diff --git a/cs/src/core/Index/PSF/README.md b/cs/src/core/Index/PSF/README.md new file mode 100644 index 000000000..cac765cd2 --- /dev/null +++ b/cs/src/core/Index/PSF/README.md @@ -0,0 +1,136 @@ +Faster Predicate Subset Functions (PSFs) +---------------------------------------- + +PSFs are based upon the [FishStore](https://github.com/microsoft/FishStore) prototype. PSFs function as secondary indexes, allowing an application to query on keys other than the single key used by the FasterKV ("Faster Key/Value store"). + + + + + +- [Overview](#overview) + * [FasterKV: Primary Index With Unique Keys](#fasterkv-primary-index-with-unique-keys) + * [PSFs: Secondary Indexes With Nonunique Keys](#psfs-secondary-indexes-with-nonunique-keys) + + [Defining Secondary Indexes](#defining-secondary-indexes) + + [Updating Secondary Indexes](#updating-secondary-indexes) + + [Querying Secondary Indexes](#querying-secondary-indexes) + + [Limitations](#limitations) + - [No Range Indexes](#no-range-indexes) + - [Fixed-Length `TPSFKey`](#fixed-length-tpsfkey) +- [Public API](#public-api) + * [The Core PSF API](#the-core-psf-api) + + [`PSFManager`](#psfmanager) + + [`IPSF`](#ipsf) + * [The FasterKV PSF API](#the-fasterkv-psf-api) + + [Registering PSFs](#registering-psfs) + + [Querying PSFs](#querying-psfs) + + [The FasterPSFSample playground app](#the-fasterpsfsample-playground-app) + + [Registering PSFs on FasterKV](#registering-psfs-on-fasterkv) + - [Registering PSFs on `Restore`](#registering-psfs-on-restore) + * [Querying PSFs on Session](#querying-psfs-on-session) +- [Code internals](#code-internals) + * [KeyPointer](#keypointer) + + + +# Overview +PSFs are "Predicate Subset Functions"; they allow defining predicates that records will match, possibly non-uniquely, for secondary indexing. PSFs are designed to be used by any data provider. Currently there is only an implementation using FasterKV as the provider, so this document will mostly reference the implementation of them as a secondary index (using "secondary FasterKVs") for a primary FasterKV store, with occasional commentary on other possible stores. + +## FasterKV: Primary Index With Unique Keys +FasterKV is essentially a hash table; as such, it has a single primary key for a given record, and there are zero or one records available for a given key. + +- An Upsert (insert or blind update) will replace an identical key, or insert a new record if an exact key match is not found +- An RMW (Read-Modify-Write) will find an exact key match and update the record, or insert a new record if an identical key match is not found +- A Read will find either a single record matching the key, or no records + +The FasterKV Key and Value may be blittable, variable length, or objects. + +## PSFs: Secondary Indexes With Nonunique Keys +PSFs implement secondary indexes by allowing the user to register a delegate that returns an alternate key for the record. For example, a record might be { Id: 42, Species: "cat" }. The "primary index" is the key inserted into the primary FasterKV; in this example it is Id, and there will be only one record with an Id of 42. A PSF might be defined for such records that returns the Species property (in C# terms, a simple lambda such as "() => this.Species;"). This return is nullable, reflecting the "predicate" terminology, which means that the record may or may not "match" the PSF; for example, a record with no pets would return null, and the record would not be stored for that PSF. This design allows zero, one, or more records to be stored for a single PSF key, entirely depending on the PSF definition. + +### Defining Secondary Indexes +PSF definition is done using the [`RegisterPSF` APIs](#registering-psfs-on-fasterkv) on the [`PSFManager`](./PSFManager.cs) class; for the FasterKV provider, a thin wrapper over these exists on the `IFasterKV` interface and is implemented by FasterKV. Each `RegisterPSF` call creates a [`PSFGroup`](./PSFGroup.cs) internally; the [`PSFGroup`](./PSFGroup.cs) contains its own FasterKV instance, and all specified PSF keys are linked in chains within that FasterKV instance. More details of this are shown below. Allowing [`PSFGroup`](./PSFGroup.cs)s has the following advantages: +- A single hashtable can be used for all PSFs, using the ordinal of the PSF as part of the hashing logic. This can save space. +- PSFs should be registered in groups where it is expected that a record will match all or none of the PSFs (that is, if a record results in a non-null key forone PSF in the group, it results in a non-null key for all PSFs in the group, and if a record results in a null key for one PSF in the group, it results in a null key for all PSFs in the group). This saves some overhead in processing variable-length composite keys in the secondary FasterKV; this [KeyPointer](#keypointer) structure is described more fully below. +- All PSFs in a [`PSFGroup`](./PSFGroup.cs) have the same `TPSFKey` type, but different groups can have different `TPSFKey` types. + +Internally, PSFs are implemented using secondary FasterKV instances with separate Key and Value types, as described in the following sections. + +### Updating Secondary Indexes +When a record is inserted into FasterKV (via Upsert or RMW), it is inserted at a unique "logical address" (essentially a page/offset combination). PSFs implement secondary indexes in Faster by allowing the user to register a delegate that returns an alternate key for the record; then the logical address of an insert (termed a RecordId; note that this is *not* the actual record value) is the value that is inserted into the secondary FasterKV instance using the alternate key. The distinction between the Key and Value defined for the primary FasterKV and the secondary Key and Value (the RecordId) is critical; the secondary FasterKV has no idea of the primary datastore's Key and Value definitions. + +Unlike the primary FasterKV's keys, the PSF keys may return multiple records. To query a PSF, the user passes a value for the alternate key; all logical addresses that were inserted with that key are returned, and then the primary FasterKV instance retrieves the actual records at those logical addresses. Whereas the primary FasterKV returns only a single record (if found) via the IFunctions callback implementation supplied by the client, PSF queries return an `IEnumerable` or `IAsyncEnumerable`. + +The logical address as the RecordId is specific the the FasterKV's use of PSFs; other data provider could provide any other record identifier that fits with their design. + +PSF updating is done by the Primary FasterKV's Upsert, RMW, or Delete operations, which call the Upsert, Update, or Delete methods on the [`PSFManager`](./PSFManager.cs). + +### Querying Secondary Indexes +The `ClientSession` object contains several overloads of `QueryPSF`, taking various combinations of [`IPSF`](./IPSF.cs), `TPSFKey` types and individual keys, and `matchPredicate`. + +The simplest query takes only a single [`IPSF`](./IPSF.cs) and `TPSFKey` key instance, returning all records that match that key for that [`IPSF`](./IPSF.cs). More complicated forms allow specifying multiple [`IPSF`](./IPSF.cs)s, multiple keys per [`IPSF`](./IPSF.cs), and multiple `TPSFKey` types (each with multiple [`IPSF`](./IPSF.cs)s, each with multiple keys). + +Because [`PSFGroup`](./PSFGroup.cs)s store `TRecordId`s, the [`PSFManager`](./PSFManager.cs) client (that is, the data provider, such as a primary FasterKV) must wrap the `QueryPSF` call with its own translation of the `TRecordId` to the actual data record; see `CreateProviderData` in `FasterPSFSessionOperations.cs`. + +The [PSFQuery API](#querying-psfs-on-session) is described in detail below. + +### Limitations +The current implementation of PSFs has some limitations. + +#### No Range Indexes +Because PSFs use hash indexes (and conceptually store multiple records for a given key as that key's collision chain), we have only equality comparisons, not ranges. This can be worked around in some cases by specifying "binned" keys; for example, a date range can be represented as bins of one minute each. In this case, the query will pass an enumeration of keys and all records for all keys will be queried; the caller must post-process each record to ensure it is within the desired range. The caller must decide the bin size, trading off the inefficiency of multiple key enumerations with the inefficiency of returning unwanted values (including the lookuup of the logical address in the primary FasterKV). + +#### Fixed-Length `TPSFKey` +PSFs currently use fixed-length keys; the `TPSFKey` type returned by a PSF execution has a type constraint of being `struct`. It must be a blittable type; the PSF API does not provide for `IVariableLengthStruct` or `SerializerSettings`, nor does it accept strings as keys. Rather than passing a string, the caller must pass some sort of string identifier, such as a hash (but do not use string.GetHashCode() for this, because it is AppDomain-specific (and also dependent on .NET version)). In this case, the hashcode becomes a "bin" of all strings (or string prefixes) corresponding to that hash code. + +# Public API +This section discusses the PSF API in more detail. There are two levels: The interface to PSFs themselves, which is [`PSFManager`](./PSFManager.cs), and how FasterKV is a client of PSFs (as well as providing code for the implementation). + +## The Core PSF API +This section describes the core PSF API on the [`PSFManager`](./PSFManager.cs) class. For examples of its use, see [The FasterKV PSF API](#the-fasterkv-psf-api), which wraps the core PSF API. + +### [`PSFManager`](./PSFManager.cs) +As discussed above, the PSF API is intended to be used by any data provider needing a hash-based index that is capable of storing a `TRecordId` from which the provider can extract its full record. However, PSF update operations must be able to execute the PSF, which requires knowledge of the provider's Key and Value types. Therefore, [`PSFManager`](./PSFManager.cs) has two generic types, both of which are opaque to PSFs: +- `TProviderData`, which is the data passed to PSF execution (the PSF must, of course, know how to operate on the provider data and form a `TPSFKey` key from it) +- `TRecordId`, which is the record identifier stored as the Value in the secondary FasterKV. + +The `TRecordId` has a type constraint of being `struct`; it must be blittable. + +### [`IPSF`](./IPSF.cs) + +## The FasterKV PSF API + +### Registering PSFs + +### Querying PSFs + +### The FasterPSFSample playground app +The [FasterPSFSample](../../../../playground/FasterPSFSample/FasterPSFSample.cs) app demonstrates registering and querying PSFs. + +### Registering PSFs on FasterKV +[Defining Secondary Indexes](#defining-secondary-indexes) provides an overview of registering PSFs. For the FasterKV provider, this is done on the [IFasterKV](../Interfaces/IFasterKV.cs) interface, because it does not do any session operations itself. TODO revisit for indexing of existing records. + +[FPSF.cs](../../../../playground/FasterPSFSample/FPSF.cs) illustrates registering the PSFs. There are several overloads, depending on the number of PSFs per group and whether the PSF can be defined by a simple lambda vs. a more complex definition (which requires an implementation of [`IPSFDefinition`](./IPSFDefinition.cs)). + +[`PSFGroup`](./PSFGroup.cs)s are not visible to the client; they are entirely internal. The return from a `RegisterPSF` call is a vector of [`IPSF`](./IPSF.cs), and various combinations of [`IPSF`](./IPSF.cs) and `TPSFKey` values are passed the the [`QueryPSF` API](#querying-secondary-indexes). + +Each [`PSFGroup`](./PSFGroup.cs)s contains its own internal "secondary" FasterKV instance, including a hash table (shared by all PSFs, including the PSF ordinal in the hash operation) and log. Because this contains hashed records for all PSFs in the group, PSFs cannot be modified once created, nor can they be added to or removed from the group individually. + +#### Registering PSFs on `Restore` +[IFasterKV](../Interfaces/IFasterKV.cs) provides a `GetRegisteredPSFNames` method that returns the names of all PSFs that were registered. Another provider would have to expose similar functionality. At `Restore` time, before any operations are done on the Primary FasterKV, the aplication must call `RegisterPSF` on those names for those groups; it *must not* change a group definition by adding, removing, or supplying a different name or functionality (lambda or definition) for a PSF in the group; doing so will break access to existing records in the group. + +If an application creates a new version of a PSF, it should encode the version information into the PSF's name, e.g. "Dog v1.1". The application must keep track internally of all PSF names and functionality (lambda or definition) for any groups it has created. + +Dropping a group is done by omitting it from the `RegisterPSF` calls done at `Restore` time. This is the only supported versioning mechanism: "drop" a group (by not registering it) and then create a new group with the updated definitions (and possibly changed PSF membership). + +## Querying PSFs on Session +[Querying Secondary Indexes](#querying-secondary-indexes) provides an overview of querying PSFs. For the FasterKV provider, these are done on the [ClientSession](FasterPSFSessionOperations.cs) class, because they must enter the session lock. + +# Code internals +This section presents a high-level overview of the PSF internal design and implementation. + +## KeyPointer +TODO + +Notes: +- generics don't have type constraint for ": nullable" (but they can for notnull) +- \ No newline at end of file diff --git a/cs/src/core/Index/PSF/RecordIterator.cs b/cs/src/core/Index/PSF/RecordIterator.cs new file mode 100644 index 000000000..4191ff540 --- /dev/null +++ b/cs/src/core/Index/PSF/RecordIterator.cs @@ -0,0 +1,381 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading.Tasks; + +namespace FASTER.core.Index.PSF +{ + /// + /// Base class to implement common functionality between sync and async versions of RecordIterator. + /// + internal abstract class RecordIteratorBase where TRecordId : IComparable + { + internal IPSF psf; + internal int psfIndex; + protected readonly TEnumerator enumerator; + + protected RecordIteratorBase(IPSF psf, int psfIndex, TEnumerator enumer) + { + this.psf = psf; + this.psfIndex = psfIndex; + this.enumerator = enumer; + } + + internal bool IsDone { get; set; } + + internal abstract TRecordId Current { get; } + + internal void GetIfLower(ref TRecordId currentLowest) + { + if (!this.IsDone && this.Current.CompareTo(currentLowest) < 0) + currentLowest = this.Current; + } + + internal bool IsMatch(TRecordId recordId) => !this.IsDone && this.Current.CompareTo(recordId) == 0; + + public override string ToString() => $"psfIdx {this.psfIndex}, current {this.Current}, isDone {this.IsDone}"; + } + + /// + /// A single PSF's stream of recordIds + /// + internal class RecordIterator : RecordIteratorBase> where TRecordId : IComparable + { + internal RecordIterator(IPSF psf, int psfIndex, IEnumerator enumerator) : base(psf, psfIndex, enumerator) { } + + internal bool Next() + { + if (!this.IsDone) + this.IsDone = !this.enumerator.MoveNext(); + return !this.IsDone; + } + + internal override TRecordId Current => this.enumerator.Current; + } + +#if DOTNETCORE + /// + /// A single PSF's async stream of recordIds + /// + internal class AsyncRecordIterator : RecordIteratorBase> where TRecordId : IComparable + { + internal AsyncRecordIterator(IPSF psf, int psfIndex, IAsyncEnumerator enumerator) : base(psf, psfIndex, enumerator) { } + + internal async Task NextAsync() + { + if (!this.IsDone) + this.IsDone = !await this.enumerator.MoveNextAsync(); + return !this.IsDone; + } + + internal override TRecordId Current => this.enumerator.Current; + } +#endif // DOTNETCORE + + /// + /// Base class to implement common functionality between sync and async versions of KeyTypeRecordIterator. + /// + internal class KeyTypeRecordIteratorBase where TRecordId : IComparable + { + private readonly int keyTypeOrdinal; + protected readonly RecordIteratorBase[] psfRecordIterators; + protected readonly PSFQuerySettings querySettings; + private int numDone; + + protected KeyTypeRecordIteratorBase(int keyTypeOrd, RecordIteratorBase[] psfRecEnums, PSFQuerySettings querySettings) + { + this.keyTypeOrdinal = keyTypeOrd; + this.psfRecordIterators = psfRecEnums; + this.querySettings = querySettings; + } + + internal int Count => this.psfRecordIterators.Length; + + internal bool IsDone => this.numDone == this.Count; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + protected static (bool @continue, TRecordId lowest, bool first) GetIfLower(RecordIteratorBase recordIter, TRecordId currentLowest, bool isFirst) + { + if (recordIter.IsDone) + return (true, currentLowest, isFirst); + if (isFirst) + return (true, recordIter.Current, false); + recordIter.GetIfLower(ref currentLowest); + return (true, currentLowest, false); + } + + protected bool ContinueOnEOS(RecordIteratorBase recordIter) + { + ++this.numDone; + return !this.querySettings.CancelOnEOS(recordIter.psf, (this.keyTypeOrdinal, recordIter.psfIndex)); + } + + internal void MarkMatchIndicators(TRecordId currentLowest, bool[] matchIndicators) + { + foreach (var (recordIter, psfIndex) in this.psfRecordIterators.Select((item, index) => (item, index))) + matchIndicators[psfIndex] = recordIter.IsMatch(currentLowest); + } + + internal IEnumerable> GetEnumeratorsMatchingPrevLowest(TRecordId previousLowest) + { + foreach (var (recordIter, psfIndex) in this.psfRecordIterators.Select((item, index) => (item, index))) + { + if (recordIter.IsDone) + continue; + if (this.querySettings.IsCanceled) + yield break; + if (recordIter.IsMatch(previousLowest)) + yield return recordIter; + } + } + + public override string ToString() => $"keyTypeOrd {this.keyTypeOrdinal}, count {this.Count}, isDone {this.IsDone}"; + } + + /// + /// A single TPSFKey type's vector of its PSFs' streams of recordIds (each TPSFKey type may have multiple PSFs being queried). + /// + internal class KeyTypeRecordIterator : KeyTypeRecordIteratorBase> where TRecordId : IComparable + { + internal KeyTypeRecordIterator(int keyTypeOrd, IPSF psf1, IEnumerator psfRecordEnumerator1, PSFQuerySettings querySettings) + : base(keyTypeOrd, new[] { new RecordIterator(psf1, 0, psfRecordEnumerator1) }, querySettings) + { } + + internal KeyTypeRecordIterator(int keyTypeOrd, IEnumerable<(IPSF psf, IEnumerator psfRecEnum)> queryResults, PSFQuerySettings querySettings) + : base(keyTypeOrd, queryResults.Select((tup, psfIdx) => new RecordIterator(tup.psf, psfIdx, tup.psfRecEnum)).ToArray(), querySettings) + { } + + #region Sync methods + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal (bool @continue, TRecordId lowest, bool first) Initialize(TRecordId currentLowest, bool isFirst) + => IterateAndGetIfLower(true, currentLowest, currentLowest, isFirst); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal (bool @continue, TRecordId lowest, bool first) GetNextLowest(TRecordId previousLowest, TRecordId currentLowest, bool isFirst) + => IterateAndGetIfLower(false, previousLowest, currentLowest, isFirst); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal (bool @continue, TRecordId lowest, bool first) IterateAndGetIfLower(bool isInit, TRecordId previousLowest, TRecordId currentLowest, bool isFirst) + { + var tuple = (true, currentLowest, isFirst); + foreach (var recordIter in this.psfRecordIterators.Where(iter => !iter.IsDone).Cast>()) + { + // If in initialization, always do the initial Next(); otherwise, advance the iterator if it matches the previous lowest record ID. + if (((isInit || recordIter.IsMatch(previousLowest)) && !recordIter.Next() && !ContinueOnEOS(recordIter)) || querySettings.IsCanceled) + return (false, currentLowest, isFirst); + tuple = GetIfLower(recordIter, tuple.currentLowest, tuple.isFirst); + } + return tuple; + } + #endregion Sync methods + } + +#if DOTNETCORE + /// + /// A single TPSFKey type's vector of its PSFs' async streams of recordIds (each TPSFKey type may have multiple PSFs being queried). + /// + internal class AsyncKeyTypeRecordIterator : KeyTypeRecordIteratorBase> where TRecordId : IComparable + { + internal AsyncKeyTypeRecordIterator(int keyTypeOrd, IPSF psf1, IAsyncEnumerator psfRecordEnumerator1, PSFQuerySettings querySettings) + : base(keyTypeOrd, new[] { new AsyncRecordIterator(psf1, 0, psfRecordEnumerator1) }, querySettings) + { } + + internal AsyncKeyTypeRecordIterator(int keyTypeOrd, IEnumerable<(IPSF psf, IAsyncEnumerator psfRecEnum)> queryResults, PSFQuerySettings querySettings) + : base(keyTypeOrd, queryResults.Select((tup, psfIdx) => new AsyncRecordIterator(tup.psf, psfIdx, tup.psfRecEnum)).ToArray(), querySettings) + { } + + #region Async methods + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal async Task<(bool @continue, TRecordId lowest, bool first)> InitializeAsync(TRecordId currentLowest, bool isFirst) + => await IterateAndGetIfLowerAsync(true, currentLowest, currentLowest, isFirst); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal async Task<(bool @continue, TRecordId lowest, bool first)> GetNextLowestAsync(TRecordId previousLowest, TRecordId currentLowest, bool isFirst) + => await IterateAndGetIfLowerAsync(false, previousLowest, currentLowest, isFirst); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal async Task<(bool @continue, TRecordId lowest, bool first)> IterateAndGetIfLowerAsync(bool isInit, TRecordId previousLowest, TRecordId currentLowest, bool isFirst) + { + var tuple = (true, currentLowest, isFirst); + foreach (var recordIter in this.psfRecordIterators.Where(iter => !iter.IsDone).Cast>()) + { + // If in initialization, always do the initial Next(); otherwise, advance the iterator if it matches the previous lowest record ID. + if (((isInit || recordIter.IsMatch(previousLowest)) && !await recordIter.NextAsync() && !ContinueOnEOS(recordIter)) || querySettings.IsCanceled) + return (false, currentLowest, isFirst); + tuple = GetIfLower(recordIter, tuple.currentLowest, tuple.isFirst); + } + return tuple; + } + #endregion Async methods + } +#endif // DOTNETCORE + + /// + /// Base class to implement common functionality between sync and async versions of KeyTypeRecordIterator. + /// + internal class QueryRecordIteratorBase where TRecordId : IComparable + { + protected readonly KeyTypeRecordIteratorBase[] keyTypeRecordIterators; + private readonly bool[][] matchIndicators; + private readonly PSFQuerySettings querySettings; + private readonly Func callerLambda; + + protected QueryRecordIteratorBase(KeyTypeRecordIteratorBase[] ktris, Func callerLambda, PSFQuerySettings querySettings) + { + this.keyTypeRecordIterators = ktris; + this.matchIndicators = this.keyTypeRecordIterators.Select(ktri => new bool[ktri.Count]).ToArray(); + this.callerLambda = callerLambda; + this.querySettings = querySettings; + } + + protected bool CallLambda(ref TRecordId current, out bool emit) + { + var allDone = true; + foreach (var (keyIter, keyIndex) in this.keyTypeRecordIterators.Select((iter, index) => (iter, index))) + { + keyIter.MarkMatchIndicators(current, this.matchIndicators[keyIndex]); + allDone &= keyIter.IsDone; + } + + allDone |= this.querySettings.IsCanceled; + emit = !allDone && this.callerLambda(this.matchIndicators); + return !allDone; + } + } + + /// + /// The complete query's PSFs' streams of recordIds (each TPSFKey type may have multiple PSFs being queried). + /// + internal class QueryRecordIterator : QueryRecordIteratorBase> where TRecordId : IComparable + { + // Unfortunately we must sort to do the merge. + private static IEnumerator GetOrderedEnumerator(IEnumerable enumerable) => enumerable.OrderBy(rec => rec).GetEnumerator(); + + internal QueryRecordIterator(IPSF psf1, IEnumerable keyRecords1, IPSF psf2, IEnumerable keyRecords2, + Func callerLambda, PSFQuerySettings querySettings) + : base(new[] { + new KeyTypeRecordIterator(0, psf1, GetOrderedEnumerator(keyRecords1), querySettings), + new KeyTypeRecordIterator(1, psf2, GetOrderedEnumerator(keyRecords2), querySettings) + }, callerLambda, querySettings) + { } + + internal QueryRecordIterator(IPSF psf1, IEnumerable keyRecords1, IPSF psf2, IEnumerable keyRecords2, + IPSF psf3, IEnumerable keyRecords3, + Func callerLambda, PSFQuerySettings querySettings) + : base(new[] { + new KeyTypeRecordIterator(0, psf1, GetOrderedEnumerator(keyRecords1), querySettings), + new KeyTypeRecordIterator(1, psf2, GetOrderedEnumerator(keyRecords2), querySettings), + new KeyTypeRecordIterator(2, psf3, GetOrderedEnumerator(keyRecords3), querySettings) + }, callerLambda, querySettings) + { } + + internal QueryRecordIterator(IEnumerable keyRecEnums)>> keyTypeQueryResultsEnum, + Func callerLambda, PSFQuerySettings querySettings) + : base(keyTypeQueryResultsEnum.Select((ktqr, index) => new KeyTypeRecordIterator(index, ktqr.Select(tuple => (tuple.psf, GetOrderedEnumerator(tuple.keyRecEnums))), querySettings)).ToArray(), + callerLambda, querySettings) + { } + + #region Sync methods + internal IEnumerable Run() + { + // The tuple is necessary due to async prohibition of byref parameters. + (bool @continue, TRecordId current, bool isFirst) tuple = (true, default, true); + foreach (var keyIter in this.keyTypeRecordIterators.Cast>()) + { + tuple = keyIter.Initialize(tuple.current, tuple.isFirst); + if (!tuple.@continue) + yield break; + } + + while (true) + { + if (!CallLambda(ref tuple.current, out bool emit)) + yield break; + if (emit) + yield return tuple.current; + + var prevLowest = tuple.current; + tuple.isFirst = true; + foreach (var keyIter in this.keyTypeRecordIterators.Cast>()) + { + // TODOperf: consider a PQ here. Given that we have to go through all matchIndicators anyway, at what number of streams would the additional complexity improve speed? + tuple = keyIter.GetNextLowest(prevLowest, tuple.current, tuple.isFirst); + if (!tuple.@continue) + yield break; + } + } + } + #endregion Sync methods + } + +#if DOTNETCORE + /// + /// The complete query's PSFs' async streams of recordIds (each TPSFKey type may have multiple PSFs being queried). + /// + internal class AsyncQueryRecordIterator : QueryRecordIteratorBase> where TRecordId : IComparable + { + // Unfortunately we must sort to do the merge. + private static IAsyncEnumerator GetOrderedEnumerator(IAsyncEnumerable enumerable) => enumerable.OrderBy(rec => rec).GetAsyncEnumerator(); + + internal AsyncQueryRecordIterator(IPSF psf1, IAsyncEnumerable keyRecords1, IPSF psf2, IAsyncEnumerable keyRecords2, + Func callerLambda, PSFQuerySettings querySettings) + : base(new[] { + new AsyncKeyTypeRecordIterator(0, psf1, GetOrderedEnumerator(keyRecords1), querySettings), + new AsyncKeyTypeRecordIterator(1, psf2, GetOrderedEnumerator(keyRecords2), querySettings) + }, callerLambda, querySettings) + { } + + internal AsyncQueryRecordIterator(IPSF psf1, IAsyncEnumerable keyRecords1, IPSF psf2, IAsyncEnumerable keyRecords2, + IPSF psf3, IAsyncEnumerable keyRecords3, + Func callerLambda, PSFQuerySettings querySettings) + : base(new[] { + new AsyncKeyTypeRecordIterator(0, psf1, GetOrderedEnumerator(keyRecords1), querySettings), + new AsyncKeyTypeRecordIterator(1, psf2, GetOrderedEnumerator(keyRecords2), querySettings), + new AsyncKeyTypeRecordIterator(2, psf3, GetOrderedEnumerator(keyRecords3), querySettings) + }, callerLambda, querySettings) + { } + + internal AsyncQueryRecordIterator(IEnumerable keyRecEnums)>> keyTypeQueryResultsEnum, + Func callerLambda, PSFQuerySettings querySettings) + : base(keyTypeQueryResultsEnum.Select((ktqr, index) => new AsyncKeyTypeRecordIterator(index, ktqr.Select(tuple => (tuple.psf, GetOrderedEnumerator(tuple.keyRecEnums))), querySettings)).ToArray(), + callerLambda, querySettings) + { } + + #region Sync methods + internal async IAsyncEnumerable Run() + { + // The tuple is necessary due to async prohibition of byref parameters. + (bool @continue, TRecordId current, bool isFirst) tuple = (true, default, true); + foreach (var keyIter in this.keyTypeRecordIterators.Cast>()) + { + tuple = await keyIter.InitializeAsync(tuple.current, tuple.isFirst); + if (!tuple.@continue) + yield break; + } + + while (true) + { + if (!CallLambda(ref tuple.current, out bool emit)) + yield break; + if (emit) + yield return tuple.current; + + var prevLowest = tuple.current; + tuple.isFirst = true; + foreach (var keyIter in this.keyTypeRecordIterators.Cast>()) + { + // TODOperf: consider a PQ here. Given that we have to go through all matchIndicators anyway, at what number of streams would the additional complexity improve speed? + tuple = await keyIter.GetNextLowestAsync(prevLowest, tuple.current, tuple.isFirst); + if (!tuple.@continue) + yield break; + } + } + } + #endregion Sync methods + } +#endif // DOTNETCORE +} diff --git a/cs/src/core/Index/PSF/SessionManager.cs b/cs/src/core/Index/PSF/SessionManager.cs new file mode 100644 index 000000000..696ca50a5 --- /dev/null +++ b/cs/src/core/Index/PSF/SessionManager.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +using System.Collections.Concurrent; +using System.Runtime.CompilerServices; + +namespace FASTER.core +{ + // TODO: How are the sessions disposed? + class SessionManager + where TKey : struct + where TValue : struct + where TFunctions : IFunctions, new() + { + private readonly ConcurrentStack> freeSessions + = new ConcurrentStack>(); + private readonly ConcurrentBag> allSessions + = new ConcurrentBag>(); + + internal FasterKV fht; + private readonly bool threadAffinitized; + + internal SessionManager(FasterKV fht, bool threadAff) + { + this.fht = fht; + this.threadAffinitized = threadAff; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal ClientSession GetSession() + { + // Sessions are used only on post-RegisterPSF actions (Upsert, RMW, Query). + if (this.freeSessions.TryPop(out var session)) + return session; + session = this.fht.NewSession(new TFunctions(), threadAffinitized: this.threadAffinitized); + this.allSessions.Add(session); + return session; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void ReleaseSession(ClientSession session) + { + // TODO: Cap on number of saved sessions? + this.freeSessions.Push(session); + } + } +}