From 96466b15603b3370ddb69bc172a0b498df98ff26 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Oct 2024 11:34:28 -0700 Subject: [PATCH 01/15] Bump mermaid from 10.9.2 to 10.9.3 in /website (#743) Bumps [mermaid](https://github.com/mermaid-js/mermaid) from 10.9.2 to 10.9.3. - [Release notes](https://github.com/mermaid-js/mermaid/releases) - [Changelog](https://github.com/mermaid-js/mermaid/blob/develop/CHANGELOG.md) - [Commits](https://github.com/mermaid-js/mermaid/compare/v10.9.2...v10.9.3) --- updated-dependencies: - dependency-name: mermaid dependency-type: indirect ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- website/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/website/yarn.lock b/website/yarn.lock index 9680f24d09..19ff61b727 100644 --- a/website/yarn.lock +++ b/website/yarn.lock @@ -6019,9 +6019,9 @@ merge2@^1.3.0, merge2@^1.4.1: integrity sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg== mermaid@^10.4.0: - version "10.9.2" - resolved "https://registry.yarnpkg.com/mermaid/-/mermaid-10.9.2.tgz#108fe98060e6fba6bc826e5b454674aa2d32b817" - integrity sha512-UkZyMSuIYcI1Q0H+2pv/5CiY84sOwQ2XlKoDZMl9Y/MtrLEtxQtyA6LWGkMxnZxj0dJqI+7nw51bYjNnrbdFsQ== + version "10.9.3" + resolved "https://registry.yarnpkg.com/mermaid/-/mermaid-10.9.3.tgz#90bc6f15c33dbe5d9507fed31592cc0d88fee9f7" + integrity sha512-V80X1isSEvAewIL3xhmz/rVmc27CVljcsbWxkxlWJWY/1kQa4XOABqpDl2qQLGKzpKm6WbTfUEKImBlUfFYArw== dependencies: "@braintree/sanitize-url" "^6.0.1" "@types/d3-scale" "^4.0.3" From eabadb5f85155277fe492ce00cfe719323aba78e Mon Sep 17 00:00:00 2001 From: Tal Zaccai Date: Tue, 22 Oct 2024 19:28:00 -0700 Subject: [PATCH 02/15] Expanding RespCommand enum to ushort (#736) * wip * fix * Fixed comments * Another fix * More fixes * format * Update libs/server/Resp/Parser/RespCommand.cs Co-authored-by: Meir Blachman --------- Co-authored-by: Meir Blachman --- libs/server/ACL/CommandPermissionSet.cs | 7 ++--- libs/server/API/GarnetApi.cs | 4 +-- libs/server/API/GarnetWatchApi.cs | 2 +- libs/server/API/IGarnetApi.cs | 4 +-- libs/server/Custom/CustomCommandManager.cs | 18 +++++++----- libs/server/Custom/CustomObjectCommand.cs | 2 +- libs/server/Custom/CustomRawStringCommand.cs | 4 +-- libs/server/Custom/CustomRespCommands.cs | 4 +-- libs/server/InputHeader.cs | 8 ++--- .../Objects/Types/GarnetObjectSerializer.cs | 4 +-- libs/server/Objects/Types/GarnetObjectType.cs | 10 +++++++ libs/server/Resp/Bitmap/BitmapCommands.cs | 20 ++++++------- .../Resp/Bitmap/BitmapManagerBitfield.cs | 10 +++---- libs/server/Resp/Parser/RespCommand.cs | 29 ++++++++++++++----- libs/server/Resp/RespCommandAccessor.cs | 2 +- libs/server/Resp/RespServerSession.cs | 2 +- .../Functions/MainStore/PrivateMethods.cs | 6 ++-- .../Storage/Functions/MainStore/RMWMethods.cs | 16 +++++----- .../Functions/MainStore/ReadMethods.cs | 8 ++--- .../Functions/MainStore/VarLenInputMethods.cs | 8 ++--- .../Functions/ObjectStore/PrivateMethods.cs | 2 +- .../Functions/ObjectStore/RMWMethods.cs | 10 +++---- .../Functions/ObjectStore/ReadMethods.cs | 2 +- .../Storage/Session/MainStore/BitmapOps.cs | 14 ++++----- test/Garnet.test/RespCommandTests.cs | 2 +- test/Garnet.test/TestProcedureBitmap.cs | 4 +-- 26 files changed, 114 insertions(+), 88 deletions(-) diff --git a/libs/server/ACL/CommandPermissionSet.cs b/libs/server/ACL/CommandPermissionSet.cs index 175a1ba5d9..fc9f0cb694 100644 --- a/libs/server/ACL/CommandPermissionSet.cs +++ b/libs/server/ACL/CommandPermissionSet.cs @@ -167,10 +167,9 @@ public bool IsEquivalentTo(CommandPermissionSet other) /// private static ushort GetCommandListLength() { - int commandCount = (int)Enum.GetValues().Where(static cmd => cmd != RespCommand.NONE && cmd != RespCommand.INVALID).Max(); - - int neededBits = commandCount; - int neededULongs = neededBits / 64; + // # of bits needed to represent all valid commands + var neededBits = (ushort)RespCommandExtensions.LastValidCommand + 1; + var neededULongs = neededBits / 64; if ((neededBits % 64) != 0) { diff --git a/libs/server/API/GarnetApi.cs b/libs/server/API/GarnetApi.cs index 4906d6d5ed..34c91d00a5 100644 --- a/libs/server/API/GarnetApi.cs +++ b/libs/server/API/GarnetApi.cs @@ -333,11 +333,11 @@ public GarnetStatus StringBitPosition(ref SpanByte key, ref RawStringInput input => storageSession.StringBitPosition(ref key, ref input, ref output, ref context); /// - public GarnetStatus StringBitField(ref SpanByte key, ref RawStringInput input, byte secondaryCommand, ref SpanByteAndMemory output) + public GarnetStatus StringBitField(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output) => storageSession.StringBitField(ref key, ref input, secondaryCommand, ref output, ref context); /// - public GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput input, byte secondaryCommand, ref SpanByteAndMemory output) + public GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output) => storageSession.StringBitFieldReadOnly(ref key, ref input, secondaryCommand, ref output, ref context); /// diff --git a/libs/server/API/GarnetWatchApi.cs b/libs/server/API/GarnetWatchApi.cs index ee0896ac56..ef6d82266f 100644 --- a/libs/server/API/GarnetWatchApi.cs +++ b/libs/server/API/GarnetWatchApi.cs @@ -473,7 +473,7 @@ public GarnetStatus StringBitPosition(ref SpanByte key, ref RawStringInput input } /// - public GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput input, byte secondaryCommand, ref SpanByteAndMemory output) + public GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output) { garnetApi.WATCH(new ArgSlice(ref key), StoreType.Main); return garnetApi.StringBitFieldReadOnly(ref key, ref input, secondaryCommand, ref output); diff --git a/libs/server/API/IGarnetApi.cs b/libs/server/API/IGarnetApi.cs index a10d0add73..e271407c63 100644 --- a/libs/server/API/IGarnetApi.cs +++ b/libs/server/API/IGarnetApi.cs @@ -987,7 +987,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// /// /// - GarnetStatus StringBitField(ref SpanByte key, ref RawStringInput input, byte secondaryCommand, ref SpanByteAndMemory output); + GarnetStatus StringBitField(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output); /// /// Performs arbitrary bitfield integer operations on strings. @@ -1622,7 +1622,7 @@ public interface IGarnetReadApi /// /// /// - GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput input, byte secondaryCommand, ref SpanByteAndMemory output); + GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output); #endregion diff --git a/libs/server/Custom/CustomCommandManager.cs b/libs/server/Custom/CustomCommandManager.cs index 07fa54224c..409b5ba344 100644 --- a/libs/server/Custom/CustomCommandManager.cs +++ b/libs/server/Custom/CustomCommandManager.cs @@ -12,8 +12,10 @@ namespace Garnet.server /// public class CustomCommandManager { - internal const byte StartOffset = 200; - internal const int MaxRegistrations = byte.MaxValue - StartOffset; + internal static readonly ushort StartOffset = (ushort)(RespCommandExtensions.LastValidCommand + 1); + internal static readonly int MaxRegistrations = ushort.MaxValue - StartOffset; + internal static readonly byte TypeIdStartOffset = (byte)(GarnetObjectTypeExtensions.LastObjectType + 1); + internal static readonly int MaxTypeRegistrations = (byte)(GarnetObjectTypeExtensions.FirstSpecialObjectType) - TypeIdStartOffset; internal readonly CustomRawStringCommand[] rawStringCommandMap; internal readonly CustomObjectCommandWrapper[] objectCommandMap; @@ -34,7 +36,7 @@ public class CustomCommandManager public CustomCommandManager() { rawStringCommandMap = new CustomRawStringCommand[MaxRegistrations]; - objectCommandMap = new CustomObjectCommandWrapper[MaxRegistrations]; + objectCommandMap = new CustomObjectCommandWrapper[MaxTypeRegistrations]; transactionProcMap = new CustomTransaction[MaxRegistrations]; // can increase up to byte.MaxValue customProcedureMap = new CustomProcedureWrapper[MaxRegistrations]; } @@ -45,7 +47,7 @@ internal int Register(string name, CommandType type, CustomRawStringFunctions cu if (id >= MaxRegistrations) throw new Exception("Out of registration space"); - rawStringCommandMap[id] = new CustomRawStringCommand(name, (byte)id, type, customFunctions, expirationTicks); + rawStringCommandMap[id] = new CustomRawStringCommand(name, (ushort)id, type, customFunctions, expirationTicks); if (commandInfo != null) CustomCommandsInfo.Add(name, commandInfo); if (commandDocs != null) CustomCommandsDocs.Add(name, commandDocs); return id; @@ -73,7 +75,7 @@ internal int RegisterType(CustomObjectFactory factory) do { type = Interlocked.Increment(ref ObjectTypeId) - 1; - if (type >= MaxRegistrations) + if (type >= MaxTypeRegistrations) throw new Exception("Out of registration space"); } while (objectCommandMap[type] != null); @@ -84,7 +86,7 @@ internal int RegisterType(CustomObjectFactory factory) internal void RegisterType(int objectTypeId, CustomObjectFactory factory) { - if (objectTypeId >= MaxRegistrations) + if (objectTypeId >= MaxTypeRegistrations) throw new Exception("Type is outside registration space"); if (ObjectTypeId <= objectTypeId) ObjectTypeId = objectTypeId + 1; @@ -106,7 +108,7 @@ internal void RegisterType(int objectTypeId, CustomObjectFactory factory) if (objectTypeId == -1) { objectTypeId = Interlocked.Increment(ref ObjectTypeId) - 1; - if (objectTypeId >= MaxRegistrations) + if (objectTypeId >= MaxTypeRegistrations) throw new Exception("Out of registration space"); objectCommandMap[objectTypeId] = new CustomObjectCommandWrapper((byte)objectTypeId, factory); } @@ -135,7 +137,7 @@ internal void RegisterType(int objectTypeId, CustomObjectFactory factory) if (objectTypeId == -1) { objectTypeId = Interlocked.Increment(ref ObjectTypeId) - 1; - if (objectTypeId >= MaxRegistrations) + if (objectTypeId >= MaxTypeRegistrations) throw new Exception("Out of registration space"); objectCommandMap[objectTypeId] = new CustomObjectCommandWrapper((byte)objectTypeId, factory); } diff --git a/libs/server/Custom/CustomObjectCommand.cs b/libs/server/Custom/CustomObjectCommand.cs index edf7610021..fac189e48f 100644 --- a/libs/server/Custom/CustomObjectCommand.cs +++ b/libs/server/Custom/CustomObjectCommand.cs @@ -24,6 +24,6 @@ internal CustomObjectCommand(string name, byte id, byte subid, CommandType type, this.functions = functions; } - internal RespCommand GetRespCommand() => (RespCommand)(id + CustomCommandManager.StartOffset); + internal GarnetObjectType GetObjectType() => (GarnetObjectType)(id + CustomCommandManager.TypeIdStartOffset); } } \ No newline at end of file diff --git a/libs/server/Custom/CustomRawStringCommand.cs b/libs/server/Custom/CustomRawStringCommand.cs index e4f7351e75..4cc4b5c425 100644 --- a/libs/server/Custom/CustomRawStringCommand.cs +++ b/libs/server/Custom/CustomRawStringCommand.cs @@ -7,12 +7,12 @@ class CustomRawStringCommand { public readonly string NameStr; public readonly byte[] name; - public readonly byte id; + public readonly ushort id; public readonly CommandType type; public readonly CustomRawStringFunctions functions; public long expirationTicks; - internal CustomRawStringCommand(string name, byte id, CommandType type, CustomRawStringFunctions functions, long expirationTicks) + internal CustomRawStringCommand(string name, ushort id, CommandType type, CustomRawStringFunctions functions, long expirationTicks) { NameStr = name.ToUpperInvariant(); this.name = System.Text.Encoding.ASCII.GetBytes(NameStr); diff --git a/libs/server/Custom/CustomRespCommands.cs b/libs/server/Custom/CustomRespCommands.cs index 1426c6e1a2..d625f66733 100644 --- a/libs/server/Custom/CustomRespCommands.cs +++ b/libs/server/Custom/CustomRespCommands.cs @@ -132,14 +132,14 @@ private bool TryCustomRawStringCommand(RespCommand cmd, long expirat /// /// Custom object command /// - private bool TryCustomObjectCommand(RespCommand cmd, byte subid, CommandType type, ref TGarnetApi storageApi) + private bool TryCustomObjectCommand(GarnetObjectType objType, byte subid, CommandType type, ref TGarnetApi storageApi) where TGarnetApi : IGarnetAdvancedApi { var keyBytes = parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); // Prepare input - var header = new RespInputHeader(cmd) { SubId = subid }; + var header = new RespInputHeader(objType) { SubId = subid }; var input = new ObjectInput(header, ref parseState, 1); var output = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(null) }; diff --git a/libs/server/InputHeader.cs b/libs/server/InputHeader.cs index 3779666538..6625b37a26 100644 --- a/libs/server/InputHeader.cs +++ b/libs/server/InputHeader.cs @@ -38,7 +38,7 @@ public struct RespInputHeader /// /// Size of header /// - public const int Size = 2; + public const int Size = 3; internal const byte FlagMask = (byte)RespInputFlags.SetGet - 1; [FieldOffset(0)] @@ -47,7 +47,7 @@ public struct RespInputHeader [FieldOffset(0)] internal GarnetObjectType type; - [FieldOffset(1)] + [FieldOffset(2)] internal RespInputFlags flags; /// @@ -77,7 +77,7 @@ public RespInputHeader(GarnetObjectType type, RespInputFlags flags = 0) /// /// Command /// Flags - public void SetHeader(byte cmd, byte flags) + public void SetHeader(ushort cmd, byte flags) { this.cmd = (RespCommand)cmd; this.flags = (RespInputFlags)flags; @@ -347,7 +347,7 @@ public RawStringInput(RespCommand cmd, RespInputFlags flags = 0, long arg1 = 0) /// Command /// Flags /// General-purpose argument - public RawStringInput(byte cmd, byte flags = 0, long arg1 = 0) : + public RawStringInput(ushort cmd, byte flags = 0, long arg1 = 0) : this((RespCommand)cmd, (RespInputFlags)flags, arg1) { diff --git a/libs/server/Objects/Types/GarnetObjectSerializer.cs b/libs/server/Objects/Types/GarnetObjectSerializer.cs index 3c72473652..2563371269 100644 --- a/libs/server/Objects/Types/GarnetObjectSerializer.cs +++ b/libs/server/Objects/Types/GarnetObjectSerializer.cs @@ -58,8 +58,8 @@ private IGarnetObject DeserializeInternal(BinaryReader binaryReader) private IGarnetObject CustomDeserialize(byte type, BinaryReader binaryReader) { - if (type < CustomCommandManager.StartOffset) return null; - return customCommands[type - CustomCommandManager.StartOffset].factory.Deserialize(type, binaryReader); + if (type < CustomCommandManager.TypeIdStartOffset) return null; + return customCommands[type - CustomCommandManager.TypeIdStartOffset].factory.Deserialize(type, binaryReader); } /// diff --git a/libs/server/Objects/Types/GarnetObjectType.cs b/libs/server/Objects/Types/GarnetObjectType.cs index 61d6122df5..69ad2e793b 100644 --- a/libs/server/Objects/Types/GarnetObjectType.cs +++ b/libs/server/Objects/Types/GarnetObjectType.cs @@ -29,6 +29,10 @@ public enum GarnetObjectType : byte /// Set, + // Any new object type inserted here should update GarnetObjectTypeExtensions.LastObjectType + + // Any new special type inserted here should update GarnetObjectTypeExtensions.FirstSpecialObjectType + /// /// Special type indicating EXPIRETIME command /// @@ -68,6 +72,12 @@ public enum GarnetObjectType : byte /// Indicating a Custom Object command /// All = 0xfb + } + + public static class GarnetObjectTypeExtensions + { + internal const GarnetObjectType LastObjectType = GarnetObjectType.Set; + internal const GarnetObjectType FirstSpecialObjectType = GarnetObjectType.ExpireTime; } } \ No newline at end of file diff --git a/libs/server/Resp/Bitmap/BitmapCommands.cs b/libs/server/Resp/Bitmap/BitmapCommands.cs index bfa7a6e081..5f536cf802 100644 --- a/libs/server/Resp/Bitmap/BitmapCommands.cs +++ b/libs/server/Resp/Bitmap/BitmapCommands.cs @@ -65,43 +65,43 @@ public struct BitFieldCmdArgs /// BITFIELD command /// [FieldOffset(0)] - public byte secondaryOpCode; + public RespCommand secondaryCommand; /// /// encoding info /// - [FieldOffset(1)] + [FieldOffset(sizeof(RespCommand))] public byte typeInfo; /// /// offset /// - [FieldOffset(2)] + [FieldOffset(sizeof(RespCommand) + sizeof(byte))] public long offset; /// /// value /// - [FieldOffset(10)] + [FieldOffset(sizeof(RespCommand) + sizeof(byte) + sizeof(long))] public long value; /// /// BitFieldOverflow enum /// - [FieldOffset(18)] + [FieldOffset(sizeof(RespCommand) + sizeof(byte) + (2 * sizeof(long)))] public byte overflowType; /// /// add a command to execute in bitfield /// - /// + /// /// /// /// /// - public BitFieldCmdArgs(byte secondaryOpCode, byte typeInfo, long offset, long value, byte overflowType) + public BitFieldCmdArgs(RespCommand secondaryCommand, byte typeInfo, long offset, long value, byte overflowType) { - this.secondaryOpCode = secondaryOpCode; + this.secondaryCommand = secondaryCommand; this.typeInfo = typeInfo; this.offset = offset; this.value = value; @@ -488,7 +488,7 @@ private bool StringBitField(ref TGarnetApi storageApi, bool readOnly for (var i = 0; i < secondaryCommandArgs.Count; i++) { - var opCode = (byte)secondaryCommandArgs[i].Item1; + var opCode = secondaryCommandArgs[i].Item1; var opArgs = secondaryCommandArgs[i].Item2; parseState.Initialize(opArgs.Length + (isOverflowTypeSet ? 1 : 0)); @@ -508,7 +508,7 @@ private bool StringBitField(ref TGarnetApi storageApi, bool readOnly var status = storageApi.StringBitField(ref sbKey, ref input, opCode, ref output); - if (status == GarnetStatus.NOTFOUND && opCode == (byte)RespCommand.GET) + if (status == GarnetStatus.NOTFOUND && opCode == RespCommand.GET) { while (!RespWriteUtils.WriteArrayItem(0, ref dcurr, dend)) SendAndReset(); diff --git a/libs/server/Resp/Bitmap/BitmapManagerBitfield.cs b/libs/server/Resp/Bitmap/BitmapManagerBitfield.cs index f33d1493cf..5c60528dcf 100644 --- a/libs/server/Resp/Bitmap/BitmapManagerBitfield.cs +++ b/libs/server/Resp/Bitmap/BitmapManagerBitfield.cs @@ -9,7 +9,7 @@ namespace Garnet.server public unsafe partial class BitmapManager { [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static byte GetBitFieldSecondaryOp(byte* input) => (*(BitFieldCmdArgs*)(input)).secondaryOpCode; + private static RespCommand GetBitFieldSecondaryOp(byte* input) => (*(BitFieldCmdArgs*)(input)).secondaryCommand; [MethodImpl(MethodImplOptions.AggressiveInlining)] private static byte GetBitFieldType(byte* input) => (*(BitFieldCmdArgs*)(input)).typeInfo; @@ -459,13 +459,13 @@ public static (long, bool) BitFieldExecute(BitFieldCmdArgs args, byte* value, in { var bitCount = (byte)(args.typeInfo & 0x7F); - switch (args.secondaryOpCode) + switch (args.secondaryCommand) { - case (byte)RespCommand.SET: + case RespCommand.SET: return SetBitfieldValue(value, valLen, args.offset, bitCount, args.typeInfo, args.value, args.overflowType); - case (byte)RespCommand.INCRBY: + case RespCommand.INCRBY: return IncrByBitfieldValue(value, valLen, args.offset, bitCount, args.typeInfo, args.value, args.overflowType); - case (byte)RespCommand.GET: + case RespCommand.GET: return (GetBitfieldValue(value, valLen, args.offset, bitCount, args.typeInfo), false); default: throw new GarnetException("BITFIELD secondary op not supported"); diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index a6b94e6b34..7cd956ebb3 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -3,6 +3,7 @@ using System; using System.Diagnostics; +using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using System.Text; @@ -14,7 +15,7 @@ namespace Garnet.server /// /// Basic RESP command enum /// - public enum RespCommand : byte + public enum RespCommand : ushort { NONE = 0x00, @@ -302,7 +303,10 @@ public enum RespCommand : byte HELLO, QUIT, // Note: Update IsNoAuth if adding new no-auth commands after this - INVALID = 0xFF, + // Max value of this enum (not including INVALID) will determine the size of RespCommand.AofIndependentBitLookup and CommandPermissionSet._commandList, + // so avoid manually setting high values unless necessary + + INVALID = 0xFFFF, } /// @@ -357,15 +361,19 @@ public static class RespCommandExtensions RespCommand.MULTI, ]; - // long is 64 bits, 4 longs accomodate 256 resp commands which is more than enough to provide a lookup for each resp command - private static readonly ulong[] AofIndepenedentBitLookup = [0, 0, 0, 0]; + private static readonly ulong[] AofIndependentBitLookup; private const int sizeOfLong = 64; // The static ctor maybe expensive but it is only ever run once, and doesn't interfere with common path static RespCommandExtensions() { - foreach (RespCommand cmd in Enum.GetValues(typeof(RespCommand))) + // # of bits needed to represent all valid commands + var maxBitsNeeded = (ushort)LastValidCommand + 1; + var lookupTableSize = (maxBitsNeeded / 64) + (maxBitsNeeded % 64 == 0 ? 0 : 1); + AofIndependentBitLookup = new ulong[lookupTableSize]; + + foreach (var cmd in Enum.GetValues()) { if (Array.IndexOf(AofIndependentCommands, cmd) == -1) continue; @@ -375,7 +383,7 @@ static RespCommandExtensions() // set the respCommand's bit to indicate int bitIdxOffset = (int)cmd % sizeOfLong; ulong bitmask = 1UL << bitIdxOffset; - AofIndepenedentBitLookup[bitIdxToUse] |= bitmask; + AofIndependentBitLookup[bitIdxToUse] |= bitmask; } } @@ -385,11 +393,13 @@ static RespCommandExtensions() [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsAofIndependent(this RespCommand cmd) { + if (cmd > LastValidCommand) return false; + // check if cmd maps to a bit vec that was set back when static ctor was run int bitIdxToUse = (int)cmd / sizeOfLong; int bitIdxOffset = (int)cmd % sizeOfLong; ulong bitmask = 1UL << bitIdxOffset; - return (AofIndepenedentBitLookup[bitIdxToUse] & bitmask) != 0; + return (AofIndependentBitLookup[bitIdxToUse] & bitmask) != 0; } /// @@ -435,6 +445,11 @@ public static ReadOnlySpan ExpandForACLs(this RespCommand cmd) internal const RespCommand LastDataCommand = RespCommand.EVALSHA; + /// + /// Last valid command (i.e. RespCommand with the largest value excluding INVALID). + /// + public static RespCommand LastValidCommand { get; } = Enum.GetValues().Where(cmd => cmd != RespCommand.INVALID).Max(); + [MethodImpl(MethodImplOptions.AggressiveInlining)] public static bool IsReadOnly(this RespCommand cmd) => cmd <= LastReadCommand; diff --git a/libs/server/Resp/RespCommandAccessor.cs b/libs/server/Resp/RespCommandAccessor.cs index 0c3b2521ee..b5a3ac927f 100644 --- a/libs/server/Resp/RespCommandAccessor.cs +++ b/libs/server/Resp/RespCommandAccessor.cs @@ -11,6 +11,6 @@ public static class RespCommandAccessor /// /// MIGRATE /// - public static byte MIGRATE => (byte)RespCommand.MIGRATE; + public static ushort MIGRATE => (ushort)RespCommand.MIGRATE; } } \ No newline at end of file diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index 91332dbdfd..ef7d28d8ac 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -807,7 +807,7 @@ bool NetworkCustomObjCmd(ref TGarnetApi storageApi) } // Perform the operation - TryCustomObjectCommand(currentCustomObjectCommand.GetRespCommand(), currentCustomObjectCommand.subid, + TryCustomObjectCommand(currentCustomObjectCommand.GetObjectType(), currentCustomObjectCommand.subid, currentCustomObjectCommand.type, ref storageApi); currentCustomObjectCommand = null; return true; diff --git a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs index 75b9a98803..50bf90392a 100644 --- a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs +++ b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs @@ -713,12 +713,12 @@ void WriteLogDelete(ref SpanByte key, long version, int sessionID) BitFieldCmdArgs GetBitFieldArguments(ref RawStringInput input) { var currTokenIdx = input.parseStateFirstArgIdx; - var opCode = (byte)input.parseState.GetEnum(currTokenIdx++, true); + var cmd = input.parseState.GetEnum(currTokenIdx++, true); var encodingArg = input.parseState.GetString(currTokenIdx++); var offsetArg = input.parseState.GetString(currTokenIdx++); long value = default; - if (opCode == (byte)RespCommand.SET || opCode == (byte)RespCommand.INCRBY) + if (cmd == RespCommand.SET || cmd == RespCommand.INCRBY) { value = input.parseState.GetLong(currTokenIdx++); } @@ -738,7 +738,7 @@ BitFieldCmdArgs GetBitFieldArguments(ref RawStringInput input) // Calculate number offset from bitCount if offsetArg starts with # var offset = offsetArg[0] == '#' ? long.Parse(offsetArg.AsSpan(1)) * bitCount : long.Parse(offsetArg); - return new BitFieldCmdArgs(opCode, typeInfo, offset, value, overflowType); + return new BitFieldCmdArgs(cmd, typeInfo, offset, value, overflowType); } } } \ No newline at end of file diff --git a/libs/server/Storage/Functions/MainStore/RMWMethods.cs b/libs/server/Storage/Functions/MainStore/RMWMethods.cs index f81c258629..58a9c5a6ac 100644 --- a/libs/server/Storage/Functions/MainStore/RMWMethods.cs +++ b/libs/server/Storage/Functions/MainStore/RMWMethods.cs @@ -30,11 +30,11 @@ public bool NeedInitialUpdate(ref SpanByte key, ref RawStringInput input, ref Sp case RespCommand.GETEX: return false; default: - if ((byte)input.header.cmd >= CustomCommandManager.StartOffset) + if ((ushort)input.header.cmd >= CustomCommandManager.StartOffset) { (IMemoryOwner Memory, int Length) outp = (output.Memory, 0); var ret = functionsState - .customCommands[(byte)input.header.cmd - CustomCommandManager.StartOffset].functions + .customCommands[(ushort)input.header.cmd - CustomCommandManager.StartOffset].functions .NeedInitialUpdate(key.AsReadOnlySpan(), ref input, ref outp); output.Memory = outp.Memory; output.Length = outp.Length; @@ -179,9 +179,9 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB default: value.UnmarkExtraMetadata(); - if ((byte)input.header.cmd >= CustomCommandManager.StartOffset) + if ((ushort)input.header.cmd >= CustomCommandManager.StartOffset) { - var functions = functionsState.customCommands[(byte)input.header.cmd - CustomCommandManager.StartOffset].functions; + var functions = functionsState.customCommands[(ushort)input.header.cmd - CustomCommandManager.StartOffset].functions; // compute metadata size for result var expiration = input.arg1; metadataSize = expiration switch @@ -507,7 +507,7 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re return false; default: - var cmd = (byte)input.header.cmd; + var cmd = (ushort)input.header.cmd; if (cmd >= CustomCommandManager.StartOffset) { var functions = functionsState.customCommands[cmd - CustomCommandManager.StartOffset].functions; @@ -576,10 +576,10 @@ public bool NeedCopyUpdate(ref SpanByte key, ref RawStringInput input, ref SpanB } return false; default: - if ((byte)input.header.cmd >= CustomCommandManager.StartOffset) + if ((ushort)input.header.cmd >= CustomCommandManager.StartOffset) { (IMemoryOwner Memory, int Length) outp = (output.Memory, 0); - var ret = functionsState.customCommands[(byte)input.header.cmd - CustomCommandManager.StartOffset].functions + var ret = functionsState.customCommands[(ushort)input.header.cmd - CustomCommandManager.StartOffset].functions .NeedCopyUpdate(key.AsReadOnlySpan(), ref input, oldValue.AsReadOnlySpan(), ref outp); output.Memory = outp.Memory; output.Length = outp.Length; @@ -812,7 +812,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte break; default: - if ((byte)input.header.cmd >= CustomCommandManager.StartOffset) + if ((ushort)input.header.cmd >= CustomCommandManager.StartOffset) { var functions = functionsState.customCommands[(byte)input.header.cmd - CustomCommandManager.StartOffset].functions; var expiration = input.arg1; diff --git a/libs/server/Storage/Functions/MainStore/ReadMethods.cs b/libs/server/Storage/Functions/MainStore/ReadMethods.cs index 798d3372e5..cd0a0be785 100644 --- a/libs/server/Storage/Functions/MainStore/ReadMethods.cs +++ b/libs/server/Storage/Functions/MainStore/ReadMethods.cs @@ -19,11 +19,11 @@ public bool SingleReader(ref SpanByte key, ref RawStringInput input, ref SpanByt return false; var cmd = input.header.cmd; - if ((byte)cmd >= CustomCommandManager.StartOffset) + if ((ushort)cmd >= CustomCommandManager.StartOffset) { var valueLength = value.LengthWithoutMetadata; (IMemoryOwner Memory, int Length) output = (dst.Memory, 0); - var ret = functionsState.customCommands[(byte)cmd - CustomCommandManager.StartOffset].functions + var ret = functionsState.customCommands[(ushort)cmd - CustomCommandManager.StartOffset].functions .Reader(key.AsReadOnlySpan(), ref input, value.AsReadOnlySpan(), ref output, ref readInfo); Debug.Assert(valueLength <= value.LengthWithoutMetadata); dst.Memory = output.Memory; @@ -50,11 +50,11 @@ public bool ConcurrentReader(ref SpanByte key, ref RawStringInput input, ref Spa } var cmd = input.header.cmd; - if ((byte)cmd >= CustomCommandManager.StartOffset) + if ((ushort)cmd >= CustomCommandManager.StartOffset) { var valueLength = value.LengthWithoutMetadata; (IMemoryOwner Memory, int Length) output = (dst.Memory, 0); - var ret = functionsState.customCommands[(byte)cmd - CustomCommandManager.StartOffset].functions + var ret = functionsState.customCommands[(ushort)cmd - CustomCommandManager.StartOffset].functions .Reader(key.AsReadOnlySpan(), ref input, value.AsReadOnlySpan(), ref output, ref readInfo); Debug.Assert(valueLength <= value.LengthWithoutMetadata); dst.Memory = output.Memory; diff --git a/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs b/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs index 8d1d0db833..e17917a546 100644 --- a/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs +++ b/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs @@ -119,9 +119,9 @@ public int GetRMWInitialValueLength(ref RawStringInput input) return sizeof(int) + ndigits; default: - if ((byte)cmd >= CustomCommandManager.StartOffset) + if ((ushort)cmd >= CustomCommandManager.StartOffset) { - var functions = functionsState.customCommands[(byte)cmd - CustomCommandManager.StartOffset].functions; + var functions = functionsState.customCommands[(ushort)cmd - CustomCommandManager.StartOffset].functions; // Compute metadata size for result int metadataSize = input.arg1 switch { @@ -236,9 +236,9 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) return sizeof(int) + t.Length + valueLength; default: - if ((byte)cmd >= CustomCommandManager.StartOffset) + if ((ushort)cmd >= CustomCommandManager.StartOffset) { - var functions = functionsState.customCommands[(byte)cmd - CustomCommandManager.StartOffset].functions; + var functions = functionsState.customCommands[(ushort)cmd - CustomCommandManager.StartOffset].functions; // compute metadata for result var metadataSize = input.arg1 switch { diff --git a/libs/server/Storage/Functions/ObjectStore/PrivateMethods.cs b/libs/server/Storage/Functions/ObjectStore/PrivateMethods.cs index 1be16e51a3..b175f7a509 100644 --- a/libs/server/Storage/Functions/ObjectStore/PrivateMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/PrivateMethods.cs @@ -184,7 +184,7 @@ static bool EvaluateObjectExpireInPlace(ExpireOption optionType, bool expiryExis [MethodImpl(MethodImplOptions.AggressiveInlining)] private CustomObjectFunctions GetCustomObjectCommand(ref ObjectInput input, GarnetObjectType type) { - var objectId = (byte)((byte)type - CustomCommandManager.StartOffset); + var objectId = (byte)((byte)type - CustomCommandManager.TypeIdStartOffset); var cmdId = input.header.SubId; var customObjectCommand = functionsState.customObjectCommands[objectId].commandMap[cmdId].functions; return customObjectCommand; diff --git a/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs b/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs index bec97eb922..9186d19b35 100644 --- a/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs @@ -26,7 +26,7 @@ public bool NeedInitialUpdate(ref byte[] key, ref ObjectInput input, ref GarnetO case GarnetObjectType.Persist: return false; default: - if ((byte)type < CustomCommandManager.StartOffset) + if ((byte)type < CustomCommandManager.TypeIdStartOffset) return GarnetObject.NeedToCreate(input.header); else { @@ -44,7 +44,7 @@ public bool NeedInitialUpdate(ref byte[] key, ref ObjectInput input, ref GarnetO public bool InitialUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetObject value, ref GarnetObjectStoreOutput output, ref RMWInfo rmwInfo, ref RecordInfo recordInfo) { var type = input.header.type; - if ((byte)type < CustomCommandManager.StartOffset) + if ((byte)type < CustomCommandManager.TypeIdStartOffset) { value = GarnetObject.Create(type); value.Operate(ref input, ref output.spanByteAndMemory, out _, out _); @@ -55,7 +55,7 @@ public bool InitialUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetObj Debug.Assert(type != GarnetObjectType.Expire && type != GarnetObjectType.PExpire && type != GarnetObjectType.Persist, "Expire and Persist commands should have been handled already by NeedInitialUpdate."); var customObjectCommand = GetCustomObjectCommand(ref input, type); - var objectId = (byte)((byte)type - CustomCommandManager.StartOffset); + var objectId = (byte)((byte)type - CustomCommandManager.TypeIdStartOffset); value = functionsState.customObjectCommands[objectId].factory.Create((byte)type); (IMemoryOwner Memory, int Length) outp = (output.spanByteAndMemory.Memory, 0); @@ -140,7 +140,7 @@ bool InPlaceUpdaterWorker(ref byte[] key, ref ObjectInput input, ref IGarnetObje CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output.spanByteAndMemory); return true; default: - if ((byte)input.header.type < CustomCommandManager.StartOffset) + if ((byte)input.header.type < CustomCommandManager.TypeIdStartOffset) { var operateSuccessful = value.Operate(ref input, ref output.spanByteAndMemory, out sizeChange, out var removeKey); @@ -232,7 +232,7 @@ public bool PostCopyUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetOb CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref output.spanByteAndMemory); break; default: - if ((byte)input.header.type < CustomCommandManager.StartOffset) + if ((byte)input.header.type < CustomCommandManager.TypeIdStartOffset) { value.Operate(ref input, ref output.spanByteAndMemory, out _, out var removeKey); if (removeKey) diff --git a/libs/server/Storage/Functions/ObjectStore/ReadMethods.cs b/libs/server/Storage/Functions/ObjectStore/ReadMethods.cs index ab6c2558af..1fdb489825 100644 --- a/libs/server/Storage/Functions/ObjectStore/ReadMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/ReadMethods.cs @@ -46,7 +46,7 @@ public bool SingleReader(ref byte[] key, ref ObjectInput input, ref IGarnetObjec return true; default: - if ((byte)input.header.type < CustomCommandManager.StartOffset) + if ((byte)input.header.type < CustomCommandManager.TypeIdStartOffset) return value.Operate(ref input, ref dst.spanByteAndMemory, out _, out _); if (IncorrectObjectType(ref input, value, ref dst.spanByteAndMemory)) diff --git a/libs/server/Storage/Session/MainStore/BitmapOps.cs b/libs/server/Storage/Session/MainStore/BitmapOps.cs index 3e7451bd90..458185191b 100644 --- a/libs/server/Storage/Session/MainStore/BitmapOps.cs +++ b/libs/server/Storage/Session/MainStore/BitmapOps.cs @@ -280,7 +280,7 @@ public unsafe GarnetStatus StringBitField(ArgSlice key, List 0 ? "i"u8 : "u"u8; var encodingSuffix = commandArguments[i].typeInfo & 0x7F; var encodingSuffixLength = NumUtils.NumDigits(encodingSuffix); @@ -339,13 +339,13 @@ public unsafe GarnetStatus StringBitField(ArgSlice key, List(ref SpanByte key, ref Raw where TContext : ITsavoriteContext => Read_MainStore(ref key, ref input, ref output, ref context); - public unsafe GarnetStatus StringBitField(ref SpanByte key, ref RawStringInput input, byte secondaryCommand, ref SpanByteAndMemory output, ref TContext context) + public unsafe GarnetStatus StringBitField(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output, ref TContext context) where TContext : ITsavoriteContext { GarnetStatus status; - if (secondaryCommand == (byte)RespCommand.GET) + if (secondaryCommand == RespCommand.GET) status = Read_MainStore(ref key, ref input, ref output, ref context); else status = RMW_MainStore(ref key, ref input, ref output, ref context); return status; } - public unsafe GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput input, byte secondaryCommand, ref SpanByteAndMemory output, ref TContext context) + public unsafe GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput input, RespCommand secondaryCommand, ref SpanByteAndMemory output, ref TContext context) where TContext : ITsavoriteContext { GarnetStatus status = GarnetStatus.NOTFOUND; - if (secondaryCommand == (byte)RespCommand.GET) + if (secondaryCommand == RespCommand.GET) status = Read_MainStore(ref key, ref input, ref output, ref context); return status; } diff --git a/test/Garnet.test/RespCommandTests.cs b/test/Garnet.test/RespCommandTests.cs index 7f75319ad1..339266f60d 100644 --- a/test/Garnet.test/RespCommandTests.cs +++ b/test/Garnet.test/RespCommandTests.cs @@ -383,7 +383,7 @@ public void AofIndependentCommandsTest() RespCommand.MULTI, ]; - foreach (RespCommand cmd in Enum.GetValues(typeof(RespCommand))) + foreach (var cmd in Enum.GetValues().Where(cmd => cmd != RespCommand.INVALID)) { var expectedAofIndependence = Array.IndexOf(aofIndpendentCmds, cmd) != -1; ClassicAssert.AreEqual(expectedAofIndependence, cmd.IsAofIndependent()); diff --git a/test/Garnet.test/TestProcedureBitmap.cs b/test/Garnet.test/TestProcedureBitmap.cs index 9712cb21cc..b9b3e3d65e 100644 --- a/test/Garnet.test/TestProcedureBitmap.cs +++ b/test/Garnet.test/TestProcedureBitmap.cs @@ -133,10 +133,10 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p api.SET(bitmapA, data); var listCommands = new List(); - var bitFieldArguments = new BitFieldCmdArgs((byte)RespCommand.GET, ((byte)BitFieldSign.UNSIGNED | 8), 0, 0, (byte)BitFieldOverflow.WRAP); + var bitFieldArguments = new BitFieldCmdArgs(RespCommand.GET, ((byte)BitFieldSign.UNSIGNED | 8), 0, 0, (byte)BitFieldOverflow.WRAP); listCommands.Add(bitFieldArguments); - bitFieldArguments = new BitFieldCmdArgs((byte)RespCommand.INCRBY, ((byte)BitFieldSign.UNSIGNED | 4), 4, 1, (byte)BitFieldOverflow.WRAP); + bitFieldArguments = new BitFieldCmdArgs(RespCommand.INCRBY, ((byte)BitFieldSign.UNSIGNED | 4), 4, 1, (byte)BitFieldOverflow.WRAP); listCommands.Add(bitFieldArguments); api.StringBitField(bitmapA, listCommands, out var resultBitField); From 8a368b56d339b2b01a6f7e2452c08b209b7b53f7 Mon Sep 17 00:00:00 2001 From: Vijay Nirmal Date: Wed, 23 Oct 2024 08:31:41 +0530 Subject: [PATCH 03/15] Fixed ZDIFF return empty array with one key, should return the orginal set (#738) Co-authored-by: Tal Zaccai --- .../Session/ObjectStore/SortedSetOps.cs | 7 ++++ test/Garnet.test/RespSortedSetTests.cs | 37 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs index a0e42fc132..a378f62001 100644 --- a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs +++ b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs @@ -568,6 +568,13 @@ public unsafe GarnetStatus SortedSetDifference(ArgSlice[] keys, out Dictionary db.SortedSetScores(keys[1], values[1])); } + [Test] + public void CanDoZDiff() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var key1 = new RedisKey("key1"); + var key2 = new RedisKey("key2"); + var key1Values = new[] { new SortedSetEntry("Hello", 1), new SortedSetEntry("World", 2) }; + var key2Values = new[] { new SortedSetEntry("Hello", 5), new SortedSetEntry("Mundo", 7) }; + var expectedValue = new SortedSetEntry("World", 2); + + db.SortedSetAdd(key1, key1Values); + db.SortedSetAdd(key2, key2Values); + + var diff = db.SortedSetCombine(SetOperation.Difference, [key1, key2]); + ClassicAssert.AreEqual(1, diff.Length); + ClassicAssert.AreEqual(expectedValue.Element.ToString(), diff[0].ToString()); + + var diffWithScore = db.SortedSetCombineWithScores(SetOperation.Difference, [key1, key2]); + ClassicAssert.AreEqual(1, diffWithScore.Length); + ClassicAssert.AreEqual(expectedValue.Element.ToString(), diffWithScore[0].Element.ToString()); + ClassicAssert.AreEqual(expectedValue.Score, diffWithScore[0].Score); + + // With only one key, it should return the same elements + diffWithScore = db.SortedSetCombineWithScores(SetOperation.Difference, [key1]); + ClassicAssert.AreEqual(2, diffWithScore.Length); + ClassicAssert.AreEqual(key1Values[0].Element.ToString(), diffWithScore[0].Element.ToString()); + ClassicAssert.AreEqual(key1Values[0].Score, diffWithScore[0].Score); + ClassicAssert.AreEqual(key1Values[1].Element.ToString(), diffWithScore[1].Element.ToString()); + ClassicAssert.AreEqual(key1Values[1].Score, diffWithScore[1].Score); + + // With no value key, it should return an empty array + diffWithScore = db.SortedSetCombineWithScores(SetOperation.Difference, [new RedisKey("key3")]); + ClassicAssert.AreEqual(0, diffWithScore.Length); + } + #endregion #region LightClientTests From 3a7c4ba4011745bd92296ee5a618e239cc9e2b73 Mon Sep 17 00:00:00 2001 From: Vasileios Zois <96085550+vazois@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:18:57 -0700 Subject: [PATCH 04/15] bump version (#745) --- .azure/pipelines/azure-pipelines-external-release.yml | 2 +- libs/host/GarnetServer.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure/pipelines/azure-pipelines-external-release.yml b/.azure/pipelines/azure-pipelines-external-release.yml index e59f629b5b..754e5243bb 100644 --- a/.azure/pipelines/azure-pipelines-external-release.yml +++ b/.azure/pipelines/azure-pipelines-external-release.yml @@ -3,7 +3,7 @@ # 1) update the name: string below (line 6) -- this is the version for the nuget package (e.g. 1.0.0) # 2) update \libs\host\GarnetServer.cs readonly string version (~line 32) -- NOTE - these two values need to be the same ###################################### -name: 1.0.33 +name: 1.0.34 trigger: branches: include: diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 31850c31cd..44f1440eb5 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -29,7 +29,7 @@ namespace Garnet public class GarnetServer : IDisposable { // IMPORTANT: Keep the version in sync with .azure\pipelines\azure-pipelines-external-release.yml line ~6. - readonly string version = "1.0.33"; + readonly string version = "1.0.34"; internal GarnetProvider Provider; From 25c7eff3848ce071314d8d50366002c1233307d6 Mon Sep 17 00:00:00 2001 From: Yoganand Rajasekaran <60369795+yrajas@users.noreply.github.com> Date: Wed, 23 Oct 2024 21:29:51 -0700 Subject: [PATCH 05/15] Rate limiter using sorted sets. (#735) * Rate limiter using sorted sets. * Update store type. * Moved custom txn to test project. * Updated test csproj. * Updated documentation. --- test/Garnet.test/Extensions/RateLimiterTxn.cs | 81 +++++++++++++++++++ test/Garnet.test/RespCustomCommandTests.cs | 36 +++++++++ website/docs/extensions/objects.md | 8 +- website/docs/extensions/procedure.md | 2 +- website/docs/extensions/raw-strings.md | 20 ++--- website/docs/extensions/transactions.md | 8 +- 6 files changed, 136 insertions(+), 19 deletions(-) create mode 100644 test/Garnet.test/Extensions/RateLimiterTxn.cs diff --git a/test/Garnet.test/Extensions/RateLimiterTxn.cs b/test/Garnet.test/Extensions/RateLimiterTxn.cs new file mode 100644 index 0000000000..e1fdc0816b --- /dev/null +++ b/test/Garnet.test/Extensions/RateLimiterTxn.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using System; +using System.Text; +using Garnet.common; +using Garnet.server; +using Tsavorite.core; + +namespace Garnet +{ + sealed class RateLimiterTxn : CustomTransactionProcedure + { + public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) + { + int offset = 0; + AddKey(GetNextArg(ref procInput, ref offset), LockType.Exclusive, true); + return true; + } + + public override unsafe void Main(TGarnetApi api, ref CustomProcedureInput procInput, ref MemoryResult output) + { + int offset = 0; + var timeStamp = DateTime.Now.Ticks; + var unixTimeInMilliSecond = timeStamp / TimeSpan.TicksPerMillisecond; + + var key = GetNextArg(ref procInput, ref offset); + var slidingWindowInMilliSecondsSlice = GetNextArg(ref procInput, ref offset); + var maxRequestsSlice = GetNextArg(ref procInput, ref offset); + + long slidingWindowInMilliSeconds; + + if (NumUtils.TryParse(slidingWindowInMilliSecondsSlice.ReadOnlySpan, out slidingWindowInMilliSeconds)) + { + var trimTime = unixTimeInMilliSecond - slidingWindowInMilliSeconds; + var status = api.SortedSetRemoveRangeByScore(key, 0.ToString(), trimTime.ToString(), out var _); + + if (status == GarnetStatus.OK || status == GarnetStatus.NOTFOUND) + { + int sortedSetLength; + api.SortedSetLength(key, out sortedSetLength); + long maxRequestsVal; + + if (NumUtils.TryParse(maxRequestsSlice.ReadOnlySpan, out maxRequestsVal)) + { + if (sortedSetLength < maxRequestsVal) + { + var unixTimeInMilliSecondBytes = Encoding.ASCII.GetBytes(unixTimeInMilliSecond.ToString()); + + fixed (byte* unixTimeInMilliSecondPtr = unixTimeInMilliSecondBytes) + { + var timeInMicroSecond = timeStamp / (TimeSpan.TicksPerMillisecond / 1000); + var timeInMicroSecondBytes = Encoding.ASCII.GetBytes(timeInMicroSecond.ToString()); + fixed (byte* timeInMicroSecondBytesPtr = timeInMicroSecondBytes) + { + api.SortedSetAdd(key, new ArgSlice(unixTimeInMilliSecondPtr, unixTimeInMilliSecondBytes.Length), new ArgSlice(timeInMicroSecondBytesPtr, timeInMicroSecondBytes.Length), out var _); + api.EXPIRE(key, TimeSpan.FromMilliseconds(slidingWindowInMilliSeconds), out var _, StoreType.Object); + } + } + + WriteSimpleString(ref output, "ALLOWED"); + return; + } + } + else + { + WriteSimpleString(ref output, "FAILED"); + return; + } + } + } + else + { + WriteSimpleString(ref output, "FAILED"); + return; + } + + WriteSimpleString(ref output, "THROTTLED"); + } + } +} \ No newline at end of file diff --git a/test/Garnet.test/RespCustomCommandTests.cs b/test/Garnet.test/RespCustomCommandTests.cs index 336897bc09..6e87237096 100644 --- a/test/Garnet.test/RespCustomCommandTests.cs +++ b/test/Garnet.test/RespCustomCommandTests.cs @@ -962,5 +962,41 @@ .. libraryPaths.Skip(1), } ClassicAssert.IsNull(resp); } + + [Test] + public void RateLimiterTest() + { + server.Register.NewTransactionProc("RATELIMIT", () => new RateLimiterTxn(), new RespCommandsInfo { Arity = 4 }); + + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + // Basic allowed entries within limit + var result = db.Execute("RATELIMIT", "key1", 1000, 5); + ClassicAssert.AreEqual("ALLOWED", result.ToString()); + + result = db.Execute("RATELIMIT", "key1", 1000, 5); + ClassicAssert.AreEqual("ALLOWED", result.ToString()); + + // Throttled test + for (var i = 0; i < 5; i++) + { + result = db.Execute("RATELIMIT", "key2", 10000, 5); + ClassicAssert.AreEqual("ALLOWED", result.ToString()); + } + + result = db.Execute("RATELIMIT", "key2", 1000, 5); + ClassicAssert.AreEqual("THROTTLED", result.ToString()); + + // Test sliding window expiration + for (var i = 0; i < 5; i++) + { + result = db.Execute("RATELIMIT", "key3", 1000, 5); + ClassicAssert.AreEqual("ALLOWED", result.ToString()); + } + Thread.Sleep(TimeSpan.FromSeconds(1)); + result = db.Execute("RATELIMIT", "key3", 1000, 5); + ClassicAssert.AreEqual("ALLOWED", result.ToString()); + } } } \ No newline at end of file diff --git a/website/docs/extensions/objects.md b/website/docs/extensions/objects.md index ccdf9bd8c8..39bac83311 100644 --- a/website/docs/extensions/objects.md +++ b/website/docs/extensions/objects.md @@ -23,15 +23,15 @@ Once the new Custom Object class implementation has been added, it also requires `CustomObjectFunctions` is the base class for all custom object commands. To develop a new one, this class has to be extended and then include the custom logic. There are three methods to be implemented in a new custom object command: -- `NeedInitialUpdate(ReadOnlyMemory key, ReadOnlySpan input, ref (IMemoryOwner, int) output)`\ +- `NeedInitialUpdate(ReadOnlyMemory key, ref ObjectInput input, ref (IMemoryOwner, int) output)`\ The `NeedInitialUpdate` determines whether a new record must be created or not given the key for the record (`key`), the user input (`input`) to be used for computing the value (`value`) for the new record. If this method returns true, a new record is created otherwise not.\ -- `Reader(ReadOnlyMemory key, ReadOnlySpan input, IGarnetObject value, ref (IMemoryOwner, int) output, ref ReadInfo readInfo);`\ +- `Reader(ReadOnlyMemory key, ref ObjectInput input, IGarnetObject value, ref (IMemoryOwner, int) output, ref ReadInfo readInfo);`\ The `Reader` method performs a record read, given the key for the record to be read (`key`), the user input for computing `output` from `value`. The `readInfo` helps control whether a record needs to be expired as part of the read operation using the `ReadAction` option. If a command is meant for updating or upserting records, this method need not be overriden as the default implementation is to throw `NotImplementedException`. -- `Updater(ReadOnlyMemory key, ReadOnlySpan input, IGarnetObject value, ref (IMemoryOwner, int) output, ref RMWInfo rmwInfo)`\ +- `Updater(ReadOnlyMemory key, ref ObjectInput input, IGarnetObject value, ref (IMemoryOwner, int) output, ref RMWInfo rmwInfo)`\ The `Updater` method makes an update for RMW or upsert, given the key (`key`), the given input (`input`), the resulting value to be inserted (`value`), the location where the result of the `input` operation on `value` is to be copied (`output`) and the reference for the record info for this record (used for locking) (`rmwInfo`). If a command is meant for a pure readonly operation, this method need not be overriden as the default implementation is to throw `NotImplementedException`. There is an optional method available, if needed: -- `InitialUpdater(ReadOnlyMemory key, ReadOnlySpan input, IGarnetObject value, ref (IMemoryOwner, int) output, ref RMWInfo rmwInfo)`\ +- `InitialUpdater(ReadOnlyMemory key, ref ObjectInput input, IGarnetObject value, ref (IMemoryOwner, int) output, ref RMWInfo rmwInfo)`\ The `InitialUpdater` is available to be overriden if any specialized processing is needed when the object is created initially. Otherwise, the default implementation of this method invokes the `Updater` method. :::tip diff --git a/website/docs/extensions/procedure.md b/website/docs/extensions/procedure.md index 3074f9430a..460a253876 100644 --- a/website/docs/extensions/procedure.md +++ b/website/docs/extensions/procedure.md @@ -12,7 +12,7 @@ Custom procedures allow adding a new non-transactional procedure and registering `CustomProcedure` is the base class for all custom procedures. To develop a new one, this class has to be extended and then include the custom logic. There is one method to be implemented in a new custom procedure: -- `Execute(TGarnetApi garnetApi, ArgSlice input, ref MemoryResult output)` +- `Execute(TGarnetApi garnetApi, ref CustomProcedureInput procInput, ref MemoryResult output)` The `Execute` method has the core logic of the custom procedure. Its implementation could process input passed in through the (`input`) parameter and perform operations on Garnet by invoking any of the APIs available on `IGarnetApi`. This method then generates the output of the procedure as well. diff --git a/website/docs/extensions/raw-strings.md b/website/docs/extensions/raw-strings.md index 193b0d8895..ef61208451 100644 --- a/website/docs/extensions/raw-strings.md +++ b/website/docs/extensions/raw-strings.md @@ -12,31 +12,31 @@ Custom raw-string extensions allows adding new functions that operate on raw str `CustomRawStringFunctions` is the base class for all custom raw-string functions. To develop a new one, this class has to be extended and then include the custom logic. There are six methods to be implemented in a new custom raw-string function: -- `GetInitialLength(ReadOnlySpan input)`:\ +- `GetInitialLength(ref RawStringInput input)`:\ The `GetInitialLength` method returns the length of the initial expected length of value when populated by RMW using given input -- `GetLength(ReadOnlySpan value, ReadOnlySpan input)`\ +- `GetLength(ReadOnlySpan value, ref RawStringInput input)`\ The `GetLength` method returns the length of resulting value object when performing RMW modification of value using given input -- `InitialUpdater(ReadOnlySpan key, ReadOnlySpan input, Span value, ref (IMemoryOwner, int) output, ref RMWInfo rmwInfo)`\ +- `InitialUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, ref (IMemoryOwner, int) output, ref RMWInfo rmwInfo)`\ The `InitialUpdater` method makes an initial update for RMW, given the key (`key`), the given input (`input`), the resulting value to be inserted (`value`), the location where the result of the `input` operation on `value` is to be copied (`output`) and the reference for the record info for this record (used for locking) (`rmwInfo`) -- `InPlaceUpdater(ReadOnlySpan key, ReadOnlySpan input, Span value, ref int valueLength, ref (IMemoryOwner, int) output, ref RMWInfo rmwInfo)`\ +- `InPlaceUpdater(ReadOnlySpan key, ref RawStringInput input, Span value, ref int valueLength, ref (IMemoryOwner, int) output, ref RMWInfo rmwInfo)`\ The `InPlaceUpdater` method performs an in-place update for RMW, given the key (`key`), the given input for computing the updated (`input`), the destination to be updated (`value`), the location where the result of the `input` operation on `value` is to be copied (`output`) and the location where -- `CopyUpdater(ReadOnlySpan key, ReadOnlySpan input, ReadOnlySpan oldValue, Span newValue, ref (IMemoryOwner, int) output, ref RMWInfo rmwInfo)`\ +- `CopyUpdater(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan oldValue, Span newValue, ref (IMemoryOwner, int) output, ref RMWInfo rmwInfo)`\ The `CopyUpdate` method performs a copy update for RMW, given the key (`key`), the given input for computing `newValue` from `oldValue` (`input`), the previous value to be copied/updated (`oldValue`), the destination to be updated (`newValue`), the location where the `newValue` is to be copied (`output`) and the reference for the record info for this record (used for locking) (`rmwInfo`) -- `Reader(ReadOnlySpan key, ReadOnlySpan input, ReadOnlySpan value, ref (IMemoryOwner, int) output, ref ReadInfo readInfo);`\ +- `Reader(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan value, ref (IMemoryOwner, int) output, ref ReadInfo readInfo);`\ The `Reader` method performs a record read, given the key for the record to be read (`key`), the user input for computing `output` from `value` (`input`), the value for the record being read (`value`), the location where `value` is to be copied (`output`) and the reference for the record info for this record (used for locking) (`rmwInfo`) There are two other optional methods to be implemented: -- `NeedInitialUpdate(ReadOnlySpan key, ReadOnlySpan input, ref (IMemoryOwner, int) output)`\ +- `NeedInitialUpdate(ReadOnlySpan key, ref RawStringInput input, ref (IMemoryOwner, int) output)`\ The `NeedInitialUpdate` determines whether copy-update for RMW should be invoked, given the key for the record (`key`), the user input to be used for computing the updated value (`value`) and the location where the result of the `input` operation is to be copied.\ ***Note:*** If this method is not overridden, it returns `true` by default -- `NeedCopyUpdate(ReadOnlySpan key, ReadOnlySpan input, ReadOnlySpan oldValue, ref (IMemoryOwner, int) output)`\ +- `NeedCopyUpdate(ReadOnlySpan key, ref RawStringInput input, ReadOnlySpan oldValue, ref (IMemoryOwner, int) output)`\ The `NeedCopyUpdate` determines whether copy-update for RMW should be invoked, given the key for the record (`key`), the user input to be used for computing the updated value (`value`), the existing value that would be copied (`oldValue`) and the location where the result of the `input` operation on `oldValue` is to be copied.\ ***Note:*** If this method is not overridden, it returns `true` by default These are the helper methods for developing custom transactions. -- `GetNextArg(ArgSlice input, ref int offset)`:\ +- `GetNextArg(ref RawStringInput input, scoped ref int offset)`:\ The `GetNextArg` method is used to retrieve the next argument from the input at the specified offset. It takes an ArgSlice parameter representing the input and a reference to an int offset. It returns an ArgSlice object representing the argument as a span. The method internally reads a pointer with a length header to extract the argument. -- `GetFirstArg(ReadOnlySpan input)`:\ +- `GetFirstArg(ref RawStringInput input)`:\ The `GetFirseArg` method is used to retrieve the argument from the input at offset 0. Registering the custom raw-string function is done on the server-side by calling the `NewCommand(string name, int numParams, CommandType type, CustomRawStringFunctions customFunctions, long expirationTicks = 0)` method on the Garnet server object's `RegisterAPI` object with its name, number of parameters, the CommandType (Read / ReadModifyWrite), an instance of the custom raw-string function class, and optionally the number of ticks for expiration.\ diff --git a/website/docs/extensions/transactions.md b/website/docs/extensions/transactions.md index f0eb2dd45a..ca7be67897 100644 --- a/website/docs/extensions/transactions.md +++ b/website/docs/extensions/transactions.md @@ -12,9 +12,9 @@ Custom transactions allows adding a new transaction and registering it with Garn `CustomTransactionProcedure` is the base class for all custom transactions. To develop a new one, this class has to be extended and then include the custom logic. There are three methods to be implemented in a new custom transaction: -- `Prepare(TGarnetReadApi api, ArgSlice input)` -- `Main(TGarnetApi api, ArgSlice input, ref MemoryResult output)` -- `Finalize(TGarnetApi api, ArgSlice input, ref MemoryResult output)` +- `Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput)` +- `Main(TGarnetApi api, ref CustomProcedureInput procInput, ref MemoryResult output)` +- `Finalize(TGarnetApi api, ref CustomProcedureInput procInput, ref MemoryResult output)` The `Prepare` method implementation must setup the keys that will be involved in the transaction using utility methods available described below. The `Main` method is where the actual operation is to be performed as the locks required for the keys setup in the `Prepare` method are already obtained. The `Main` method then generates the output of the transaction as well. After the unlock of keys comes the `Finalize` phase, which can contain any non-transactional read and write operations on the store, and can write output as well. `Finalize` allows users to author complex non-transactional scripts as well: `Prepare` should simply return false, while `Main` is left unimplemented. @@ -23,7 +23,7 @@ These are the helper methods for developing custom transactions. - `RewindScratchBuffer(ref ArgSlice slice)` This method is responsible for rewinding (popping) the last entry of the scratch buffer if it contains the given ArgSlice. It takes a reference to an ArgSlice parameter and returns a boolean value indicating whether the rewind operation was successful. - `CreateArgSlice(ReadOnlySpan bytes)` This method is used to create an ArgSlice in the scratch buffer from a given ReadOnlySpan\. It takes a ReadOnlySpan\ parameter representing the argument and returns an ArgSlice object. - `CreateArgSlice(string str)` This method is similar to the previous one, but it creates an ArgSlice in UTF8 format from a given string. It takes a string parameter and returns an ArgSlice object. -- `GetNextArg(ArgSlice input, ref int offset)` This method is used to retrieve the next argument from the input at the specified offset. It takes an ArgSlice parameter representing the input and a reference to an int offset. It returns an ArgSlice object representing the argument as a span. The method internally reads a pointer with a length header to extract the argument. +- `GetNextArg(ref CustomProcedureInput procInput, ref int offset)` This method is used to retrieve the next argument from the input at the specified offset. It takes an ArgSlice parameter representing the input and a reference to an int offset. It returns an ArgSlice object representing the argument as a span. The method internally reads a pointer with a length header to extract the argument. These member functions provide utility and convenience methods for manipulating and working with the transaction data, scratch buffer, and input arguments within the CustomTransactionProcedure class. **NOTE** When invoking APIs on `IGarnetApi` multiple times with large outputs, it is possible to exhaust the internal buffer capacity. If such usage scenarios are expected, the buffer could be reset as described below. From 26f628c08819a1809ce94f932c57a84d7aa37686 Mon Sep 17 00:00:00 2001 From: Vijay Nirmal Date: Fri, 25 Oct 2024 00:23:46 +0530 Subject: [PATCH 06/15] Fixed CustomCommand test case falure after ushort conversion (#748) --- libs/server/Storage/Functions/MainStore/RMWMethods.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/server/Storage/Functions/MainStore/RMWMethods.cs b/libs/server/Storage/Functions/MainStore/RMWMethods.cs index 58a9c5a6ac..6870eba54f 100644 --- a/libs/server/Storage/Functions/MainStore/RMWMethods.cs +++ b/libs/server/Storage/Functions/MainStore/RMWMethods.cs @@ -814,7 +814,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte default: if ((ushort)input.header.cmd >= CustomCommandManager.StartOffset) { - var functions = functionsState.customCommands[(byte)input.header.cmd - CustomCommandManager.StartOffset].functions; + var functions = functionsState.customCommands[(ushort)input.header.cmd - CustomCommandManager.StartOffset].functions; var expiration = input.arg1; if (expiration == 0) { From 5a0aa99f12e4bc9cf0a19daa14109701b214567d Mon Sep 17 00:00:00 2001 From: Hamdaan Khalid <42720645+hamdaankhalid@users.noreply.github.com> Date: Thu, 24 Oct 2024 14:49:48 -0700 Subject: [PATCH 07/15] Fix disposal caused Null ref (#747) Co-authored-by: Hamdaan Khalid Co-authored-by: Badrish Chandramouli Co-authored-by: Tal Zaccai --- .../Metrics/Latency/GarnetLatencyMetrics.cs | 35 ++++++++++++++----- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/libs/server/Metrics/Latency/GarnetLatencyMetrics.cs b/libs/server/Metrics/Latency/GarnetLatencyMetrics.cs index b37c531018..034284c181 100644 --- a/libs/server/Metrics/Latency/GarnetLatencyMetrics.cs +++ b/libs/server/Metrics/Latency/GarnetLatencyMetrics.cs @@ -37,6 +37,9 @@ void Init() public void Return() { + if (metrics == null) + return; + foreach (var cmd in defaultLatencyTypes) { metrics[(int)cmd].Return(); @@ -46,7 +49,8 @@ public void Return() public void Merge(GarnetLatencyMetricsSession lm) { - if (lm.metrics == null) return; + // Metrics can be null if we are shutting down the server but there are still remaining resp server session being disposed. Early return to handle graceful exit during server disposal. + if (lm.metrics == null || metrics == null) return; int ver = lm.PriorVersion; // Use prior version for merge for (int i = 0; i < metrics.Length; i++) if (lm.metrics[i].latency[ver].TotalCount > 0) @@ -55,14 +59,19 @@ public void Merge(GarnetLatencyMetricsSession lm) public void Reset(LatencyMetricsType cmd) { + // Early return to handle graceful exit during server disposal. + if (metrics == null) + return; + int idx = (int)cmd; metrics[idx].latency.Reset(); } private List GetPercentiles(int idx) { - if (metrics[idx].latency.TotalCount == 0) + if (metrics == null || metrics[idx].latency.TotalCount == 0) return new(); + var curr = metrics[idx].latency; @@ -119,7 +128,9 @@ private List GetPercentiles(int idx) public bool GetRespHistogram(int idx, out string response, LatencyMetricsType eventType) { response = ""; - if (metrics[idx].latency.TotalCount == 0) + + // Early return to handle graceful exit during server disposal. + if (metrics == null || metrics[idx].latency.TotalCount == 0) return false; var p = GetPercentiles(idx); @@ -150,20 +161,28 @@ public string GetRespHistograms(HashSet events) { int cmdCount = 0; string response = ""; - foreach (var eventType in events) + + if (metrics != null) { - int idx = (int)eventType; - if (GetRespHistogram(idx, out var cmdHistogram, eventType)) + foreach (var eventType in events) { - response += cmdHistogram; - cmdCount++; + int idx = (int)eventType; + if (GetRespHistogram(idx, out var cmdHistogram, eventType)) + { + response += cmdHistogram; + cmdCount++; + } } } + return cmdCount == 0 ? "*0\r\n" : $"*{cmdCount * 2}\r\n" + response; } public MetricsItem[] GetLatencyMetrics(LatencyMetricsType latencyMetricsType) { + if (metrics == null) + return []; + int idx = (int)latencyMetricsType; return GetPercentiles(idx)?.ToArray(); } From 348259591ac7387025b22788b9c3c4e3744feb5a Mon Sep 17 00:00:00 2001 From: darrenge Date: Thu, 24 Oct 2024 15:35:24 -0700 Subject: [PATCH 08/15] Security fix - Setting up config files during mirroring (#749) * Updated mirror pipeline to cherry pick / push config files for nuget and npm to ADO repo to satisfy Central Feed Services (CFS) security policy. * Removed the --force from the push * Putting --force back --- .azure/pipelines/azure-pipelines-mirror.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.azure/pipelines/azure-pipelines-mirror.yml b/.azure/pipelines/azure-pipelines-mirror.yml index f7649c2138..b2e1cf6b0b 100644 --- a/.azure/pipelines/azure-pipelines-mirror.yml +++ b/.azure/pipelines/azure-pipelines-mirror.yml @@ -40,8 +40,11 @@ jobs: # Reset the main branch to match the origin/main branch # git reset --hard origin/main - # Push the updated main branch to Azure DevOps - git -c http.extraheader="AUTHORIZATION: bearer $(System.AccessToken)" push https://dev.azure.com/msresearch/_git/Garnet main + # Push the updated main branch to Azure DevOps - make sure to get the config files each push so compliant with security policy + git -c http.extraheader="AUTHORIZATION: bearer $(System.AccessToken)" remote add vso https://dev.azure.com/msresearch/_git/Garnet + git -c http.extraheader="AUTHORIZATION: bearer $(System.AccessToken)" fetch vso cfs-remediation + git -c http.extraheader="AUTHORIZATION: bearer $(System.AccessToken)" cherry-pick 0bae2b8a7d3dc9bfc59d25acf30b470ff97f6a7f + git -c http.extraheader="AUTHORIZATION: bearer $(System.AccessToken)" push --force https://dev.azure.com/msresearch/_git/Garnet main displayName: 'Mirror GitHub main to Azure DevOps main' env: From d17f4899adf7a058b1057c347a2659341521b67e Mon Sep 17 00:00:00 2001 From: Vijay Nirmal Date: Fri, 25 Oct 2024 07:25:52 +0530 Subject: [PATCH 09/15] [Compatibility] Added GEOSEARCHSTORE command and bug fix in GEOSEARCH command (#726) * Added GEOSEARCHSTORE command and bug fix in GEOSEARCH command * Fixed build * Review fix --------- Co-authored-by: Yoganand Rajasekaran <60369795+yrajas@users.noreply.github.com> --- libs/common/RespReadUtils.cs | 76 +++++- libs/resources/RespCommandsDocs.json | 221 ++++++++++++++++++ libs/resources/RespCommandsInfo.json | 38 +++ libs/server/API/GarnetApiObjectCommands.cs | 4 + libs/server/API/IGarnetApi.cs | 9 + .../SortedSetGeo/SortedSetGeoObjectImpl.cs | 76 ++++-- libs/server/Resp/CmdStrings.cs | 3 + .../Resp/Objects/SortedSetGeoCommands.cs | 42 ++++ libs/server/Resp/Parser/RespCommand.cs | 5 + libs/server/Resp/Parser/SessionParseState.cs | 15 ++ libs/server/Resp/RespServerSession.cs | 1 + .../Session/ObjectStore/SortedSetGeoOps.cs | 158 +++++++++++++ .../CommandInfoUpdater/SupportedCommand.cs | 1 + .../RedirectTests/BaseCommand.cs | 26 +++ .../ClusterSlotVerificationTests.cs | 7 + test/Garnet.test/Resp/ACL/RespCommandTests.cs | 16 ++ test/Garnet.test/RespSortedSetGeoTests.cs | 130 ++++++++++- website/docs/commands/api-compatibility.md | 2 +- website/docs/commands/data-structures.md | 19 ++ 19 files changed, 825 insertions(+), 24 deletions(-) diff --git a/libs/common/RespReadUtils.cs b/libs/common/RespReadUtils.cs index 03b7e3b77f..ed5fe2d71c 100644 --- a/libs/common/RespReadUtils.cs +++ b/libs/common/RespReadUtils.cs @@ -729,6 +729,22 @@ public static bool ReadBoolWithLengthHeader(out bool result, ref byte* ptr, byte /// The current end of the RESP message. /// True if a RESP string was successfully read. public static bool ReadStringWithLengthHeader(out string result, ref byte* ptr, byte* end) + { + ReadSpanWithLengthHeader(out var resultSpan, ref ptr, end); + result = Encoding.UTF8.GetString(resultSpan); + return true; + } + + /// + /// Tries to read a RESP-formatted string as span including its length header from the given ASCII-encoded + /// RESP message and, if successful, moves the given ptr to the end of the string value. + /// NOTE: We use ReadUnsignedLengthHeader because server does not accept $-1\r\n headers + /// + /// If parsing was successful, contains the extracted string value. + /// The starting position in the RESP message. Will be advanced if parsing is successful. + /// The current end of the RESP message. + /// True if a RESP string was successfully read. + public static bool ReadSpanWithLengthHeader(out ReadOnlySpan result, ref byte* ptr, byte* end) { result = null; @@ -753,7 +769,7 @@ public static bool ReadStringWithLengthHeader(out string result, ref byte* ptr, RespParsingException.ThrowUnexpectedToken(*(ptr - 2)); } - result = Encoding.UTF8.GetString(new ReadOnlySpan(keyPtr, length)); + result = new ReadOnlySpan(keyPtr, length); return true; } @@ -858,10 +874,40 @@ public static bool ReadErrorAsString(out string result, ref byte* ptr, byte* end return ReadString(out result, ref ptr, end); } + /// + /// Read error as span + /// + public static bool TryReadErrorAsSpan(out ReadOnlySpan result, ref byte* ptr, byte* end) + { + result = null; + if (ptr + 2 >= end) + return false; + + // Error strings need to start with a '-' + if (*ptr != '-') + { + return false; + } + + ptr++; + + return ReadAsSapn(out result, ref ptr, end); + } + /// /// Read integer as string /// public static bool ReadIntegerAsString(out string result, ref byte* ptr, byte* end) + { + var success = ReadIntegerAsSpan(out var resultSpan, ref ptr, end); + result = Encoding.UTF8.GetString(resultSpan); + return success; + } + + /// + /// Read integer as string + /// + public static bool ReadIntegerAsSpan(out ReadOnlySpan result, ref byte* ptr, byte* end) { result = null; if (ptr + 2 >= end) @@ -875,7 +921,7 @@ public static bool ReadIntegerAsString(out string result, ref byte* ptr, byte* e ptr++; - return ReadString(out result, ref ptr, end); + return ReadAsSapn(out result, ref ptr, end); } /// @@ -1033,6 +1079,32 @@ public static bool ReadString(out string result, ref byte* ptr, byte* end) return false; } + /// + /// Read ASCII string as span without header until string terminator ('\r\n'). + /// + public static bool ReadAsSapn(out ReadOnlySpan result, ref byte* ptr, byte* end) + { + result = null; + + if (ptr + 1 >= end) + return false; + + var start = ptr; + + while (ptr < end - 1) + { + if (*(ushort*)ptr == MemoryMarshal.Read("\r\n"u8)) + { + result = new ReadOnlySpan(start, (int)(ptr - start)); + ptr += 2; + return true; + } + ptr++; + } + + return false; + } + /// /// Read serialized data for migration /// diff --git a/libs/resources/RespCommandsDocs.json b/libs/resources/RespCommandsDocs.json index 2315aff5b0..0b6237ce84 100644 --- a/libs/resources/RespCommandsDocs.json +++ b/libs/resources/RespCommandsDocs.json @@ -2216,6 +2216,227 @@ } ] }, + { + "Command": "GEOSEARCHSTORE", + "Name": "GEOSEARCHSTORE", + "Summary": "Queries a geospatial index for members inside an area of a box or a circle, optionally stores the result.", + "Group": "Geo", + "Complexity": "O(N\u002Blog(M)) where N is the number of elements in the grid-aligned bounding box area around the shape provided as the filter and M is the number of items inside the shape", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "DESTINATION", + "DisplayText": "destination", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "SOURCE", + "DisplayText": "source", + "Type": "Key", + "KeySpecIndex": 1 + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "FROM", + "Type": "OneOf", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MEMBER", + "DisplayText": "member", + "Type": "String", + "Token": "FROMMEMBER" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "FROMLONLAT", + "Type": "Block", + "Token": "FROMLONLAT", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "LONGITUDE", + "DisplayText": "longitude", + "Type": "Double" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "LATITUDE", + "DisplayText": "latitude", + "Type": "Double" + } + ] + } + ] + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "BY", + "Type": "OneOf", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "CIRCLE", + "Type": "Block", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "RADIUS", + "DisplayText": "radius", + "Type": "Double", + "Token": "BYRADIUS" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "UNIT", + "Type": "OneOf", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "M", + "DisplayText": "m", + "Type": "PureToken", + "Token": "M" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "KM", + "DisplayText": "km", + "Type": "PureToken", + "Token": "KM" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "FT", + "DisplayText": "ft", + "Type": "PureToken", + "Token": "FT" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MI", + "DisplayText": "mi", + "Type": "PureToken", + "Token": "MI" + } + ] + } + ] + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "BOX", + "Type": "Block", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "WIDTH", + "DisplayText": "width", + "Type": "Double", + "Token": "BYBOX" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "HEIGHT", + "DisplayText": "height", + "Type": "Double" + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "UNIT", + "Type": "OneOf", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "M", + "DisplayText": "m", + "Type": "PureToken", + "Token": "M" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "KM", + "DisplayText": "km", + "Type": "PureToken", + "Token": "KM" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "FT", + "DisplayText": "ft", + "Type": "PureToken", + "Token": "FT" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "MI", + "DisplayText": "mi", + "Type": "PureToken", + "Token": "MI" + } + ] + } + ] + } + ] + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "ORDER", + "Type": "OneOf", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "ASC", + "DisplayText": "asc", + "Type": "PureToken", + "Token": "ASC" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "DESC", + "DisplayText": "desc", + "Type": "PureToken", + "Token": "DESC" + } + ] + }, + { + "TypeDiscriminator": "RespCommandContainerArgument", + "Name": "COUNT-BLOCK", + "Type": "Block", + "ArgumentFlags": "Optional", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "COUNT", + "DisplayText": "count", + "Type": "Integer", + "Token": "COUNT" + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "ANY", + "DisplayText": "any", + "Type": "PureToken", + "Token": "ANY", + "ArgumentFlags": "Optional" + } + ] + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "STOREDIST", + "DisplayText": "storedist", + "Type": "PureToken", + "Token": "STOREDIST", + "ArgumentFlags": "Optional" + } + ] + }, { "Command": "GET", "Name": "GET", diff --git a/libs/resources/RespCommandsInfo.json b/libs/resources/RespCommandsInfo.json index 9ff0a95864..258561bb5a 100644 --- a/libs/resources/RespCommandsInfo.json +++ b/libs/resources/RespCommandsInfo.json @@ -1276,6 +1276,44 @@ } ] }, + { + "Command": "GEOSEARCHSTORE", + "Name": "GEOSEARCHSTORE", + "Arity": -8, + "Flags": "DenyOom, Write", + "FirstKey": 1, + "LastKey": 2, + "Step": 1, + "AclCategories": "Geo, Slow, Write", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "OW, Update" + }, + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 2 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "RO, Access" + } + ] + }, { "Command": "GET", "Name": "GET", diff --git a/libs/server/API/GarnetApiObjectCommands.cs b/libs/server/API/GarnetApiObjectCommands.cs index e045363cb3..3690e93599 100644 --- a/libs/server/API/GarnetApiObjectCommands.cs +++ b/libs/server/API/GarnetApiObjectCommands.cs @@ -145,6 +145,10 @@ public GarnetStatus GeoAdd(byte[] key, ref ObjectInput input, ref GarnetObjectSt public GarnetStatus GeoCommands(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter) => storageSession.GeoCommands(key, ref input, ref outputFooter, ref objectContext); + /// + public GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref ObjectInput input, ref SpanByteAndMemory output) + => storageSession.GeoSearchStore(key, destinationKey, ref input, ref output, ref objectContext); + #endregion #region List Methods diff --git a/libs/server/API/IGarnetApi.cs b/libs/server/API/IGarnetApi.cs index e271407c63..2a4ca7a3fd 100644 --- a/libs/server/API/IGarnetApi.cs +++ b/libs/server/API/IGarnetApi.cs @@ -482,6 +482,15 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// GarnetStatus GeoAdd(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter); + /// + /// Geospatial search and store in destination key. + /// + /// + /// + /// + /// + GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref ObjectInput input, ref SpanByteAndMemory output); + #endregion #region Set Methods diff --git a/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs b/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs index feee1c7e8e..ff0526721e 100644 --- a/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs +++ b/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs @@ -437,6 +437,14 @@ private void GeoSearch(ref ObjectInput input, ref SpanByteAndMemory output) return; } + // Not supported options in Garnet: WITHHASH + if (opts.WithHash) + { + while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_GENERIC_UNK_CMD, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + return; + } + // Get the results // FROMMEMBER if (opts.FromMember && sortedSetDict.TryGetValue(fromMember, out var centerPointScore)) @@ -475,30 +483,64 @@ private void GeoSearch(ref ObjectInput input, ref SpanByteAndMemory output) } } - // Write results - while (!RespWriteUtils.WriteArrayLength(responseData.Count, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - - foreach (var item in responseData) + if (responseData.Count == 0) { - while (!RespWriteUtils.WriteBulkString(item.Member, ref curr, end)) + while (!RespWriteUtils.WriteInteger(0, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + } + else + { + var innerArrayLength = 1; + if (opts.WithDist) + { + innerArrayLength++; + } + if (opts.WithHash) + { + innerArrayLength++; + } + if (opts.WithCoord) + { + innerArrayLength++; + } - var distanceValue = (byBoxUnits.Length == 1 && (byBoxUnits[0] == (int)'M' || byBoxUnits[0] == (int)'m')) ? item.Distance - : server.GeoHash.ConvertMetersToUnits(item.Distance, byBoxUnits); - - while (!RespWriteUtils.TryWriteDoubleBulkString(distanceValue, ref curr, end)) + // Write results + while (!RespWriteUtils.WriteArrayLength(responseData.Count, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - // Write array of 2 values - while (!RespWriteUtils.WriteArrayLength(2, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + foreach (var item in responseData) + { + if (innerArrayLength > 1) + { + while (!RespWriteUtils.WriteArrayLength(innerArrayLength, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + } - while (!RespWriteUtils.TryWriteDoubleBulkString(item.Coordinates.Longitude, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + while (!RespWriteUtils.WriteBulkString(item.Member, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - while (!RespWriteUtils.TryWriteDoubleBulkString(item.Coordinates.Latitude, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + if (opts.WithDist) + { + var distanceValue = (byBoxUnits.Length == 1 && (byBoxUnits[0] == (int)'M' || byBoxUnits[0] == (int)'m')) ? item.Distance + : server.GeoHash.ConvertMetersToUnits(item.Distance, byBoxUnits); + + while (!RespWriteUtils.TryWriteDoubleBulkString(distanceValue, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + } + + if (opts.WithCoord) + { + // Write array of 2 values + while (!RespWriteUtils.WriteArrayLength(2, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + while (!RespWriteUtils.TryWriteDoubleBulkString(item.Coordinates.Longitude, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + while (!RespWriteUtils.TryWriteDoubleBulkString(item.Coordinates.Latitude, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + } + } } } } diff --git a/libs/server/Resp/CmdStrings.cs b/libs/server/Resp/CmdStrings.cs index 4815c1c8e9..87e429bc33 100644 --- a/libs/server/Resp/CmdStrings.cs +++ b/libs/server/Resp/CmdStrings.cs @@ -104,6 +104,9 @@ static partial class CmdStrings public static ReadOnlySpan rank => "rank"u8; public static ReadOnlySpan MAXLEN => "MAXLEN"u8; public static ReadOnlySpan maxlen => "maxlen"u8; + public static ReadOnlySpan STOREDIST => "STOREDIST"u8; + public static ReadOnlySpan WITHDIST => "WITHDIST"u8; + public static ReadOnlySpan WITHHASH => "WITHHASH"u8; /// /// Response strings diff --git a/libs/server/Resp/Objects/SortedSetGeoCommands.cs b/libs/server/Resp/Objects/SortedSetGeoCommands.cs index 1d36fa6001..abf9dbdccd 100644 --- a/libs/server/Resp/Objects/SortedSetGeoCommands.cs +++ b/libs/server/Resp/Objects/SortedSetGeoCommands.cs @@ -143,5 +143,47 @@ private unsafe bool GeoCommands(RespCommand command, ref TGarnetApi return true; } + + /// + /// GEOSEARCHSTORE: Store the the members of a sorted set populated with geospatial data, which are within the borders of the area specified by a given shape. + /// + /// + /// + /// + private unsafe bool GeoSearchStore(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count < 4) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.GEOSEARCHSTORE)); + } + + var destinationKey = parseState.GetArgSliceByRef(0); + var sourceKey = parseState.GetArgSliceByRef(1); + + var input = new ObjectInput(new RespInputHeader + { + type = GarnetObjectType.SortedSet, + }, ref parseState, 2); + + var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var status = storageApi.GeoSearchStore(sourceKey, destinationKey, ref input, ref output); + + switch (status) + { + case GarnetStatus.OK: + if (!output.IsSpanByte) + SendAndReset(output.Memory, output.Length); + else + dcurr += output.Length; + break; + case GarnetStatus.WRONGTYPE: + while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); + break; + } + + return true; + } } } \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index 7cd956ebb3..5b13c43fec 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -97,6 +97,7 @@ public enum RespCommand : ushort FLUSHALL, FLUSHDB, GEOADD, + GEOSEARCHSTORE, GETDEL, GETEX, GETSET, @@ -1415,6 +1416,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.ZREMRANGEBYLEX; } + else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nGEOSEA"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("RCHSTORE"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("\r\n"u8)) + { + return RespCommand.GEOSEARCHSTORE; + } break; case 15: diff --git a/libs/server/Resp/Parser/SessionParseState.cs b/libs/server/Resp/Parser/SessionParseState.cs index ee916fc860..3975062d30 100644 --- a/libs/server/Resp/Parser/SessionParseState.cs +++ b/libs/server/Resp/Parser/SessionParseState.cs @@ -162,6 +162,21 @@ public void InitializeWithArguments(ArgSlice[] args) } } + /// + /// Initialize the parse state with a given set of arguments + /// + /// Set of arguments to initialize buffer with + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void InitializeWithArguments(Span args) + { + Initialize(args.Length); + + for (var i = 0; i < args.Length; i++) + { + buffer[i] = args[i]; + } + } + /// /// Set argument at a specific index /// diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index ef7d28d8ac..426120dc61 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -611,6 +611,7 @@ private bool ProcessArrayCommands(RespCommand cmd, ref TGarnetApi st RespCommand.GEODIST => GeoCommands(cmd, ref storageApi), RespCommand.GEOPOS => GeoCommands(cmd, ref storageApi), RespCommand.GEOSEARCH => GeoCommands(cmd, ref storageApi), + RespCommand.GEOSEARCHSTORE => GeoSearchStore(ref storageApi), //HLL Commands RespCommand.PFADD => HyperLogLogAdd(ref storageApi), RespCommand.PFMERGE => HyperLogLogMerge(ref storageApi), diff --git a/libs/server/Storage/Session/ObjectStore/SortedSetGeoOps.cs b/libs/server/Storage/Session/ObjectStore/SortedSetGeoOps.cs index 00a3edf742..479745c2b1 100644 --- a/libs/server/Storage/Session/ObjectStore/SortedSetGeoOps.cs +++ b/libs/server/Storage/Session/ObjectStore/SortedSetGeoOps.cs @@ -2,6 +2,9 @@ // Licensed under the MIT license. using System; +using System.Buffers; +using System.Diagnostics; +using Garnet.common; using Tsavorite.core; namespace Garnet.server @@ -41,5 +44,160 @@ public GarnetStatus GeoCommands(byte[] key, ref ObjectInput inpu where TObjectContext : ITsavoriteContext => ReadObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref outputFooter); + /// + /// Geospatial search and store in destination key. + /// + /// + /// + /// + /// + /// + /// + public unsafe GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destination, ref ObjectInput input, ref SpanByteAndMemory output, ref TObjectContext objectContext) + where TObjectContext : ITsavoriteContext + { + var createTransaction = false; + + if (txnManager.state != TxnState.Running) + { + Debug.Assert(txnManager.state == TxnState.None); + createTransaction = true; + txnManager.SaveKeyEntryToLock(destination, true, LockType.Exclusive); + txnManager.SaveKeyEntryToLock(key, true, LockType.Shared); + _ = txnManager.Run(true); + } + var objectStoreLockableContext = txnManager.ObjectStoreLockableContext; + + var isMemory = false; + MemoryHandle ptrHandle = default; + var ptr = output.SpanByte.ToPointer(); + var curr = ptr; + var end = curr + output.Length; + + try + { + var isStoreDist = false; + Span geoSearchParseState = stackalloc ArgSlice[input.parseState.Count - input.parseStateFirstArgIdx + 1]; + var currArgIdx = 0; + var i = input.parseStateFirstArgIdx; + while (i < input.parseState.Count) + { + if (!isStoreDist && input.parseState.GetArgSliceByRef(i).ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.STOREDIST)) + { + isStoreDist = true; + break; + } + else + { + geoSearchParseState[currArgIdx] = input.parseState.GetArgSliceByRef(i); + currArgIdx++; + } + i++; + } + geoSearchParseState[currArgIdx++] = isStoreDist ? ArgSlice.FromPinnedSpan(CmdStrings.WITHDIST) : ArgSlice.FromPinnedSpan(CmdStrings.WITHHASH); + + var sourceKey = key.ToArray(); + var parseState = new SessionParseState(); + parseState.InitializeWithArguments(geoSearchParseState.Slice(0, currArgIdx)); + + var searchInput = new ObjectInput(new RespInputHeader + { + type = GarnetObjectType.SortedSet, + SortedSetOp = SortedSetOperation.GEOSEARCH, + }, ref parseState, 0); + + SpanByteAndMemory searchOutMem = default; + var searchOut = new GarnetObjectStoreOutput { spanByteAndMemory = searchOutMem }; + var status = GeoCommands(sourceKey, ref searchInput, ref searchOut, ref objectStoreLockableContext); + searchOutMem = searchOut.spanByteAndMemory; + + if (status == GarnetStatus.WRONGTYPE) + { + return GarnetStatus.WRONGTYPE; + } + + if (status == GarnetStatus.NOTFOUND) + { + _ = EXPIRE(destination, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, ref lockableContext, ref objectStoreLockableContext); + while (!RespWriteUtils.WriteInteger(0, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + return GarnetStatus.OK; + } + + Debug.Assert(!searchOutMem.IsSpanByte, "Output should not be in SpanByte format when the status is OK"); + + var searchOutHandler = searchOutMem.Memory.Memory.Pin(); + try + { + + var searchOutPtr = (byte*)searchOutHandler.Pointer; + var currOutPtr = searchOutPtr; + var endOutPtr = searchOutPtr + searchOutMem.Length; + + if (RespReadUtils.TryReadErrorAsSpan(out var error, ref currOutPtr, endOutPtr)) + { + while (!RespWriteUtils.WriteError(error, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + return GarnetStatus.OK; + } + + var destinationKey = destination.ToArray(); + objectStoreLockableContext.Delete(ref destinationKey); + + RespReadUtils.ReadUnsignedArrayLength(out var foundItems, ref currOutPtr, endOutPtr); + + // Prepare the parse state for sorted set add + var zParseState = new SessionParseState(); + zParseState.Initialize(foundItems * 2); + + for (int j = 0; j < foundItems; j++) + { + RespReadUtils.ReadUnsignedArrayLength(out var innerLength, ref currOutPtr, endOutPtr); + Debug.Assert(innerLength == 2, "Should always has location and hash or distance"); + + RespReadUtils.TrySliceWithLengthHeader(out var location, ref currOutPtr, endOutPtr); + if (isStoreDist) + { + RespReadUtils.ReadSpanWithLengthHeader(out var score, ref currOutPtr, endOutPtr); + zParseState.SetArgument(2 * j, ArgSlice.FromPinnedSpan(score)); + zParseState.SetArgument((2 * j) + 1, ArgSlice.FromPinnedSpan(location)); + } + else + { + RespReadUtils.ReadIntegerAsSpan(out var score, ref currOutPtr, endOutPtr); + zParseState.SetArgument(2 * j, ArgSlice.FromPinnedSpan(score)); + zParseState.SetArgument((2 * j) + 1, ArgSlice.FromPinnedSpan(location)); + } + } + + // Prepare the input + var zAddInput = new ObjectInput(new RespInputHeader + { + type = GarnetObjectType.SortedSet, + SortedSetOp = SortedSetOperation.ZADD, + }, ref zParseState, 0); + + var zAddOutput = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(null) }; + RMWObjectStoreOperationWithOutput(destinationKey, ref zAddInput, ref objectStoreLockableContext, ref zAddOutput); + + while (!RespWriteUtils.WriteInteger(foundItems, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + } + finally + { + searchOutHandler.Dispose(); + } + + return GarnetStatus.OK; + } + finally + { + if (createTransaction) + txnManager.Commit(true); + + if (isMemory) ptrHandle.Dispose(); + output.Length = (int)(curr - ptr); + } + } } } \ No newline at end of file diff --git a/playground/CommandInfoUpdater/SupportedCommand.cs b/playground/CommandInfoUpdater/SupportedCommand.cs index 5ccf20af5e..ca140e91b0 100644 --- a/playground/CommandInfoUpdater/SupportedCommand.cs +++ b/playground/CommandInfoUpdater/SupportedCommand.cs @@ -124,6 +124,7 @@ public class SupportedCommand new("GEOHASH", RespCommand.GEOHASH), new("GEOPOS", RespCommand.GEOPOS), new("GEOSEARCH", RespCommand.GEOSEARCH), + new("GEOSEARCHSTORE", RespCommand.GEOSEARCHSTORE), new("GET", RespCommand.GET), new("GETEX", RespCommand.GETEX), new("GETBIT", RespCommand.GETBIT), diff --git a/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs b/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs index dcd12f1f6a..3c3bd7ed4a 100644 --- a/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs +++ b/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs @@ -347,6 +347,32 @@ public override string[] GetSingleSlotRequest() public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); } + internal class GEOSEARCHSTORE : BaseCommand + { + public override bool IsArrayCommand => false; + public override bool ArrayResponse => false; + public override string Command => nameof(GEOSEARCHSTORE); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + return [ssk[0], ssk[1], "FROMMEMBER", "bar", "BYBOX", "800", "800", "km", "STOREDIST"]; + } + + public override string[] GetCrossSlotRequest() + { + var csk = GetCrossSlotKeys; + return [csk[0], csk[1], "FROMMEMBER", "bar", "BYBOX", "800", "800", "km", "STOREDIST"]; + } + + public override ArraySegment[] SetupSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + var setup = new ArraySegment[] { new ArraySegment([ssk[0], ssk[1], "FROMMEMBER", "bar", "BYBOX", "800", "800", "km", "STOREDIST"]) }; + return setup; + } + } + internal class SETRANGE : BaseCommand { public override bool IsArrayCommand => false; diff --git a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs b/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs index 429eb6543e..2e75f29dad 100644 --- a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs +++ b/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs @@ -90,6 +90,7 @@ public class ClusterSlotVerificationTests new SRANDMEMBER(), new GEOADD(), new GEOHASH(), + new GEOSEARCHSTORE(), new ZADD(), new ZREM(), new ZCARD(), @@ -267,6 +268,7 @@ public virtual void OneTimeTearDown() [TestCase("SRANDMEMBER")] [TestCase("GEOADD")] [TestCase("GEOHASH")] + [TestCase("GEOSEARCHSTORE")] [TestCase("ZADD")] [TestCase("ZREM")] [TestCase("ZCARD")] @@ -404,6 +406,7 @@ void GarnetClientSessionClusterDown(BaseCommand command) [TestCase("SRANDMEMBER")] [TestCase("GEOADD")] [TestCase("GEOHASH")] + [TestCase("GEOSEARCHSTORE")] [TestCase("ZADD")] [TestCase("ZREM")] [TestCase("ZCARD")] @@ -551,6 +554,7 @@ void GarnetClientSessionOK(BaseCommand command) [TestCase("SRANDMEMBER")] [TestCase("GEOADD")] [TestCase("GEOHASH")] + [TestCase("GEOSEARCHSTORE")] [TestCase("ZADD")] [TestCase("ZREM")] [TestCase("ZCARD")] @@ -690,6 +694,7 @@ void GarnetClientSessionCrossslotTest(BaseCommand command) [TestCase("SRANDMEMBER")] [TestCase("GEOADD")] [TestCase("GEOHASH")] + [TestCase("GEOSEARCHSTORE")] [TestCase("ZADD")] [TestCase("ZREM")] [TestCase("ZCARD")] @@ -836,6 +841,7 @@ void GarnetClientSessionMOVEDTest(BaseCommand command) [TestCase("SRANDMEMBER")] [TestCase("GEOADD")] [TestCase("GEOHASH")] + [TestCase("GEOSEARCHSTORE")] [TestCase("ZADD")] [TestCase("ZREM")] [TestCase("ZCARD")] @@ -999,6 +1005,7 @@ void GarnetClientSessionASKTest(BaseCommand command) [TestCase("SRANDMEMBER")] [TestCase("GEOADD")] [TestCase("GEOHASH")] + [TestCase("GEOSEARCHSTORE")] [TestCase("ZADD")] [TestCase("ZREM")] [TestCase("ZCARD")] diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/Garnet.test/Resp/ACL/RespCommandTests.cs index c71932497f..4f0f684219 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/Garnet.test/Resp/ACL/RespCommandTests.cs @@ -5480,6 +5480,22 @@ static async Task DoGeoSearchAsync(GarnetClient client) } } + [Test] + public async Task GeoSearchStoreACLsAsync() + { + await CheckCommandsAsync( + "GEOSEARCHSTORE", + [DoGeoSearchStoreAsync], + skipPermitted: true + ); + + static async Task DoGeoSearchStoreAsync(GarnetClient client) + { + var val = await client.ExecuteForLongResultAsync("GEOSEARCHSTORE", ["bar", "foo", "FROMMEMBER", "bar", "BYBOX", "2", "2", "M", "STOREDIST"]); + ClassicAssert.AreEqual(0, val); + } + } + [Test] public async Task ZAddACLsAsync() { diff --git a/test/Garnet.test/RespSortedSetGeoTests.cs b/test/Garnet.test/RespSortedSetGeoTests.cs index 3c855a50e0..b7c6d1cc88 100644 --- a/test/Garnet.test/RespSortedSetGeoTests.cs +++ b/test/Garnet.test/RespSortedSetGeoTests.cs @@ -274,6 +274,128 @@ public void CheckGeoSortedSetOperationsOnWrongTypeObjectSE() RespTestsUtils.CheckCommandOnWrongTypeObjectSE(() => db.GeoSearch(keys[0], values[0][1], new GeoSearchBox(800, 800, GeoUnit.Kilometers))); } + [Test] + public void CanUseGeoSearch() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var entries = new GeoEntry[cities.GetLength(0)]; + var key = new RedisKey("cities"); + var destinationKey = new RedisKey("newCities"); + for (int j = 0; j < cities.GetLength(0); j++) + { + entries[j] = new GeoEntry( + double.Parse(cities[j, 0], CultureInfo.InvariantCulture), + double.Parse(cities[j, 1], CultureInfo.InvariantCulture), + new RedisValue(cities[j, 2])); + } + var response = db.GeoAdd(key, entries, CommandFlags.None); + + var res = db.GeoSearch(key, new RedisValue("Washington"), new GeoSearchBox(800, 800, GeoUnit.Kilometers), options: GeoRadiusOptions.None); + ClassicAssert.AreEqual(3, res.Length); + ClassicAssert.AreEqual("Washington", (string)res[0].Member); + ClassicAssert.AreEqual(res[0].Distance, null); + ClassicAssert.AreEqual(res[0].Position, null); + ClassicAssert.AreEqual("Philadelphia", (string)res[1].Member); + ClassicAssert.AreEqual(res[1].Distance, null); + ClassicAssert.AreEqual(res[1].Position, null); + ClassicAssert.AreEqual("New York", (string)res[2].Member); + ClassicAssert.AreEqual(res[2].Distance, null); + ClassicAssert.AreEqual(res[2].Position, null); + + res = db.GeoSearch(key, new RedisValue("Washington"), new GeoSearchBox(800, 800, GeoUnit.Kilometers), options: GeoRadiusOptions.WithDistance); + ClassicAssert.AreEqual(3, res.Length); + ClassicAssert.AreEqual("Washington", (string)res[0].Member); + Assert.That(res[0].Distance, Is.EqualTo(0).Within(1.0 / Math.Pow(10, 6))); + ClassicAssert.AreEqual(res[0].Position, null); + ClassicAssert.AreEqual("Philadelphia", (string)res[1].Member); + Assert.That(res[1].Distance, Is.EqualTo(198.424300439725).Within(1.0 / Math.Pow(10, 6))); + ClassicAssert.AreEqual(res[1].Position, null); + ClassicAssert.AreEqual("New York", (string)res[2].Member); + Assert.That(res[2].Distance, Is.EqualTo(327.676458633557).Within(1.0 / Math.Pow(10, 6))); + ClassicAssert.AreEqual(res[2].Position, null); + + res = db.GeoSearch(key, new RedisValue("Washington"), new GeoSearchBox(800, 800, GeoUnit.Kilometers), options: GeoRadiusOptions.WithCoordinates); + ClassicAssert.AreEqual(3, res.Length); + ClassicAssert.AreEqual("Washington", (string)res[0].Member); + ClassicAssert.AreEqual(res[0].Distance, null); + Assert.That(res[0].Position.Value.Longitude, Is.EqualTo(-77.03687042).Within(1.0 / Math.Pow(10, 6))); + Assert.That(res[0].Position.Value.Latitude, Is.EqualTo(38.9071919).Within(1.0 / Math.Pow(10, 6))); + ClassicAssert.AreEqual("Philadelphia", (string)res[1].Member); + ClassicAssert.AreEqual(res[1].Distance, null); + Assert.That(res[1].Position.Value.Longitude, Is.EqualTo(-75.1652196).Within(1.0 / Math.Pow(10, 6))); + Assert.That(res[1].Position.Value.Latitude, Is.EqualTo(39.95258287).Within(1.0 / Math.Pow(10, 6))); + ClassicAssert.AreEqual("New York", (string)res[2].Member); + ClassicAssert.AreEqual(res[2].Distance, null); + Assert.That(res[2].Position.Value.Longitude, Is.EqualTo(-74.00594205).Within(1.0 / Math.Pow(10, 6))); + Assert.That(res[2].Position.Value.Latitude, Is.EqualTo(40.71278259).Within(1.0 / Math.Pow(10, 6))); + + res = db.GeoSearch(key, new RedisValue("Washington"), new GeoSearchBox(800, 800, GeoUnit.Kilometers), options: GeoRadiusOptions.WithDistance | GeoRadiusOptions.WithCoordinates); + ClassicAssert.AreEqual(3, res.Length); + ClassicAssert.AreEqual("Washington", (string)res[0].Member); + Assert.That(res[0].Distance, Is.EqualTo(0).Within(1.0 / Math.Pow(10, 6))); + Assert.That(res[0].Position.Value.Longitude, Is.EqualTo(-77.03687042).Within(1.0 / Math.Pow(10, 6))); + Assert.That(res[0].Position.Value.Latitude, Is.EqualTo(38.9071919).Within(1.0 / Math.Pow(10, 6))); + ClassicAssert.AreEqual("Philadelphia", (string)res[1].Member); + Assert.That(res[1].Distance, Is.EqualTo(198.424300439725).Within(1.0 / Math.Pow(10, 6))); + Assert.That(res[1].Position.Value.Longitude, Is.EqualTo(-75.1652196).Within(1.0 / Math.Pow(10, 6))); + Assert.That(res[1].Position.Value.Latitude, Is.EqualTo(39.95258287).Within(1.0 / Math.Pow(10, 6))); + ClassicAssert.AreEqual("New York", (string)res[2].Member); + Assert.That(res[2].Distance, Is.EqualTo(327.676458633557).Within(1.0 / Math.Pow(10, 6))); + Assert.That(res[2].Position.Value.Longitude, Is.EqualTo(-74.00594205).Within(1.0 / Math.Pow(10, 6))); + Assert.That(res[2].Position.Value.Latitude, Is.EqualTo(40.71278259).Within(1.0 / Math.Pow(10, 6))); + } + + [Test] + public void CanUseGeoSearchStore() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var entries = new GeoEntry[cities.GetLength(0)]; + var key = new RedisKey("cities"); + var destinationKey = new RedisKey("newCities"); + for (int j = 0; j < cities.GetLength(0); j++) + { + entries[j] = new GeoEntry( + double.Parse(cities[j, 0], CultureInfo.InvariantCulture), + double.Parse(cities[j, 1], CultureInfo.InvariantCulture), + new RedisValue(cities[j, 2])); + } + db.GeoAdd(key, entries, CommandFlags.None); + + db.SortedSetAdd(destinationKey, "OldValue", 10); // Add a value to be replaced + + var actualCount = db.GeoSearchAndStore(key, destinationKey, new RedisValue("Washington"), new GeoSearchBox(800, 800, GeoUnit.Kilometers), storeDistances: true); + ClassicAssert.AreEqual(3, actualCount); + + var actualValues = db.SortedSetRangeByScoreWithScores(destinationKey); + ClassicAssert.AreEqual(3, actualValues.Length); + ClassicAssert.AreEqual("Washington", (string)actualValues[0].Element); + Assert.That(actualValues[0].Score, Is.EqualTo(0).Within(1.0 / Math.Pow(10, 6))); + ClassicAssert.AreEqual("Philadelphia", (string)actualValues[1].Element); + Assert.That(actualValues[1].Score, Is.EqualTo(198.424300439725).Within(1.0 / Math.Pow(10, 6))); + ClassicAssert.AreEqual("New York", (string)actualValues[2].Element); + Assert.That(actualValues[2].Score, Is.EqualTo(327.676458633557).Within(1.0 / Math.Pow(10, 6))); + } + + [Test] + public void CanUseGeoSearchStoreWithDeleteKeyWhenSourceNotFound() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var entries = new GeoEntry[cities.GetLength(0)]; + var key = new RedisKey("cities"); + var destinationKey = new RedisKey("newCities"); + + db.SortedSetAdd(destinationKey, "OldValue", 10); + + var actualCount = db.GeoSearchAndStore(key, destinationKey, new RedisValue("Washington"), new GeoSearchBox(800, 800, GeoUnit.Kilometers), storeDistances: true); + ClassicAssert.AreEqual(0, actualCount); + + var actualValues = db.SortedSetRangeByScoreWithScores(destinationKey); + ClassicAssert.AreEqual(0, actualValues.Length); + } + //end region of SE tests #endregion @@ -301,14 +423,14 @@ public void CanUseGeoSearchWithCities(int bytesSent) //TODO: Assert values for latitude and longitude //TODO: Review precision to use for all framework versions using var lightClientRequest = TestUtils.CreateRequest(); - var responseBuf = lightClientRequest.SendCommands("GEOSEARCH cities FROMMEMBER Washington BYBOX 800 800 km WITHCOORD WITHDIST WITHHASH", "PING", 16); - var expectedResponse = "*3\r\n$10\r\nWashington\r\n$1\r\n0\r\n*2\r\n$12\r\n-77.03687042\r\n$10\r\n38.9071919\r\n$12\r\nPhiladelphia\r\n$16\r\n198.424300439725\r\n*2\r\n$11\r\n-75.1652196\r\n$11\r\n39.95258287\r\n$8\r\nNew York\r\n$16\r\n327.676458633557\r\n*2\r\n$12\r\n-74.00594205\r\n$11\r\n40.71278259\r\n+PONG\r\n"; + var responseBuf = lightClientRequest.SendCommands("GEOSEARCH cities FROMMEMBER Washington BYBOX 800 800 km WITHCOORD WITHDIST", "PING"); + var expectedResponse = "*3\r\n*3\r\n$10\r\nWashington\r\n$1\r\n0\r\n*2\r\n$17\r\n-77.0368704199791\r\n$17\r\n38.90719190239906\r\n*3\r\n$12\r\nPhiladelphia\r\n$17\r\n198.4242996738795\r\n*2\r\n$18\r\n-75.16521960496902\r\n$18\r\n39.952582865953445\r\n*3\r\n$8\r\nNew York\r\n$18\r\n327.67645879712575\r\n*2\r\n$17\r\n-74.0059420466423\r\n$18\r\n40.712782591581345\r\n+PONG\r\n"; var actualValue = Encoding.ASCII.GetString(responseBuf).Substring(0, expectedResponse.Length); ClassicAssert.IsTrue(actualValue.IndexOf("Washington") != -1); //Send command in chunks - responseBuf = lightClientRequest.SendCommandChunks("GEOSEARCH cities FROMMEMBER Washington BYBOX 800 800 km COUNT 3 ANY WITHCOORD WITHDIST WITHHASH", bytesSent, 16); - expectedResponse = "*3\r\n$10\r\nWashington\r\n$1\r\n0\r\n*2\r\n$12\r\n-77.03687042\r\n$10\r\n38.9071919\r\n$12\r\nPhiladelphia\r\n$16\r\n198.424300439725\r\n*2\r\n$11\r\n-75.1652196\r\n$11\r\n39.95258287\r\n$8\r\nNew York\r\n$16\r\n327.676458633557\r\n*2\r\n$12\r\n-74.00594205\r\n$11\r\n40.71278259\r\n+PONG\r\n"; + responseBuf = lightClientRequest.SendCommandChunks("GEOSEARCH cities FROMMEMBER Washington BYBOX 800 800 km COUNT 3 ANY WITHCOORD WITHDIST", bytesSent, 16); + expectedResponse = "*3\r\n*3\r\n$10\r\nWashington\r\n$1\r\n0\r\n*2\r\n$17\r\n-77.0368704199791\r\n$17\r\n38.90719190239906\r\n*3\r\n$12\r\nPhiladelphia\r\n$17\r\n198.4242996738795\r\n*2\r\n$18\r\n-75.16521960496902\r\n$18\r\n39.952582865953445\r\n*3\r\n$8\r\nNew York\r\n$18\r\n327.67645879712575\r\n*2\r\n$17\r\n-74.0059420466423\r\n$18\r\n40.712782591581345\r\n+PONG\r\n"; actualValue = Encoding.ASCII.GetString(responseBuf).Substring(0, expectedResponse.Length); ClassicAssert.IsTrue(actualValue.IndexOf("Washington") != -1); } diff --git a/website/docs/commands/api-compatibility.md b/website/docs/commands/api-compatibility.md index fa6b2bb6d8..8d694d3ffa 100644 --- a/website/docs/commands/api-compatibility.md +++ b/website/docs/commands/api-compatibility.md @@ -160,7 +160,7 @@ Note that this list is subject to change as we continue to expand our API comman | | GEORADIUSBYMEMBER | ➖ | (Deprecated) | | | GEORADIUSBYMEMBER_RO | ➖ | (Deprecated) | | | [GEOSEARCH](data-structures.md#geosearch) | ➕ | Partially Implemented | -| | GEOSEARCHSTORE | ➖ | | +| | [GEOSEARCHSTORE](data-structures.md#geosearchstore) | ➕ | Partially Implemented | | **HASH** | [HDEL](data-structures.md#hdel) | ➕ | | | | [HEXISTS](data-structures.md#hexists) | ➕ | | | | HEXPIRE | ➖ | | diff --git a/website/docs/commands/data-structures.md b/website/docs/commands/data-structures.md index 7c5a4d4a4c..0d4dad004a 100644 --- a/website/docs/commands/data-structures.md +++ b/website/docs/commands/data-structures.md @@ -1174,3 +1174,22 @@ An Array reply of matched members, where each sub-array represents a single item --- +### GEOSEARCHSTORE + +#### Syntax + +```bash +GEOSEARCHSTORE destination source + | BYBOX width height > [ASC | DESC] [COUNT count + [ANY]] [STOREDIST] +``` + +This command is like [GEOSEARCH](#geosearch), but stores the result in destination key. + +**Reply** + +Integer reply: the number of elements in the resulting set + +--- + From 9a08ff3e82fc8b853c7c9dc53984b245e64b792d Mon Sep 17 00:00:00 2001 From: Tal Zaccai Date: Thu, 24 Oct 2024 20:57:42 -0700 Subject: [PATCH 10/15] =?UTF-8?q?Revert=20"[Compatibility]=20Added=20GEOSE?= =?UTF-8?q?ARCHSTORE=20command=20and=20bug=20fix=20in=20GEOSEARCH=E2=80=A6?= =?UTF-8?q?"=20(#751)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit d17f4899adf7a058b1057c347a2659341521b67e. --- libs/common/RespReadUtils.cs | 76 +----- libs/resources/RespCommandsDocs.json | 221 ------------------ libs/resources/RespCommandsInfo.json | 38 --- libs/server/API/GarnetApiObjectCommands.cs | 4 - libs/server/API/IGarnetApi.cs | 9 - .../SortedSetGeo/SortedSetGeoObjectImpl.cs | 76 ++---- libs/server/Resp/CmdStrings.cs | 3 - .../Resp/Objects/SortedSetGeoCommands.cs | 42 ---- libs/server/Resp/Parser/RespCommand.cs | 5 - libs/server/Resp/Parser/SessionParseState.cs | 15 -- libs/server/Resp/RespServerSession.cs | 1 - .../Session/ObjectStore/SortedSetGeoOps.cs | 158 ------------- .../CommandInfoUpdater/SupportedCommand.cs | 1 - .../RedirectTests/BaseCommand.cs | 26 --- .../ClusterSlotVerificationTests.cs | 7 - test/Garnet.test/Resp/ACL/RespCommandTests.cs | 16 -- test/Garnet.test/RespSortedSetGeoTests.cs | 130 +---------- website/docs/commands/api-compatibility.md | 2 +- website/docs/commands/data-structures.md | 19 -- 19 files changed, 24 insertions(+), 825 deletions(-) diff --git a/libs/common/RespReadUtils.cs b/libs/common/RespReadUtils.cs index ed5fe2d71c..03b7e3b77f 100644 --- a/libs/common/RespReadUtils.cs +++ b/libs/common/RespReadUtils.cs @@ -729,22 +729,6 @@ public static bool ReadBoolWithLengthHeader(out bool result, ref byte* ptr, byte /// The current end of the RESP message. /// True if a RESP string was successfully read. public static bool ReadStringWithLengthHeader(out string result, ref byte* ptr, byte* end) - { - ReadSpanWithLengthHeader(out var resultSpan, ref ptr, end); - result = Encoding.UTF8.GetString(resultSpan); - return true; - } - - /// - /// Tries to read a RESP-formatted string as span including its length header from the given ASCII-encoded - /// RESP message and, if successful, moves the given ptr to the end of the string value. - /// NOTE: We use ReadUnsignedLengthHeader because server does not accept $-1\r\n headers - /// - /// If parsing was successful, contains the extracted string value. - /// The starting position in the RESP message. Will be advanced if parsing is successful. - /// The current end of the RESP message. - /// True if a RESP string was successfully read. - public static bool ReadSpanWithLengthHeader(out ReadOnlySpan result, ref byte* ptr, byte* end) { result = null; @@ -769,7 +753,7 @@ public static bool ReadSpanWithLengthHeader(out ReadOnlySpan result, ref b RespParsingException.ThrowUnexpectedToken(*(ptr - 2)); } - result = new ReadOnlySpan(keyPtr, length); + result = Encoding.UTF8.GetString(new ReadOnlySpan(keyPtr, length)); return true; } @@ -874,40 +858,10 @@ public static bool ReadErrorAsString(out string result, ref byte* ptr, byte* end return ReadString(out result, ref ptr, end); } - /// - /// Read error as span - /// - public static bool TryReadErrorAsSpan(out ReadOnlySpan result, ref byte* ptr, byte* end) - { - result = null; - if (ptr + 2 >= end) - return false; - - // Error strings need to start with a '-' - if (*ptr != '-') - { - return false; - } - - ptr++; - - return ReadAsSapn(out result, ref ptr, end); - } - /// /// Read integer as string /// public static bool ReadIntegerAsString(out string result, ref byte* ptr, byte* end) - { - var success = ReadIntegerAsSpan(out var resultSpan, ref ptr, end); - result = Encoding.UTF8.GetString(resultSpan); - return success; - } - - /// - /// Read integer as string - /// - public static bool ReadIntegerAsSpan(out ReadOnlySpan result, ref byte* ptr, byte* end) { result = null; if (ptr + 2 >= end) @@ -921,7 +875,7 @@ public static bool ReadIntegerAsSpan(out ReadOnlySpan result, ref byte* pt ptr++; - return ReadAsSapn(out result, ref ptr, end); + return ReadString(out result, ref ptr, end); } /// @@ -1079,32 +1033,6 @@ public static bool ReadString(out string result, ref byte* ptr, byte* end) return false; } - /// - /// Read ASCII string as span without header until string terminator ('\r\n'). - /// - public static bool ReadAsSapn(out ReadOnlySpan result, ref byte* ptr, byte* end) - { - result = null; - - if (ptr + 1 >= end) - return false; - - var start = ptr; - - while (ptr < end - 1) - { - if (*(ushort*)ptr == MemoryMarshal.Read("\r\n"u8)) - { - result = new ReadOnlySpan(start, (int)(ptr - start)); - ptr += 2; - return true; - } - ptr++; - } - - return false; - } - /// /// Read serialized data for migration /// diff --git a/libs/resources/RespCommandsDocs.json b/libs/resources/RespCommandsDocs.json index 0b6237ce84..2315aff5b0 100644 --- a/libs/resources/RespCommandsDocs.json +++ b/libs/resources/RespCommandsDocs.json @@ -2216,227 +2216,6 @@ } ] }, - { - "Command": "GEOSEARCHSTORE", - "Name": "GEOSEARCHSTORE", - "Summary": "Queries a geospatial index for members inside an area of a box or a circle, optionally stores the result.", - "Group": "Geo", - "Complexity": "O(N\u002Blog(M)) where N is the number of elements in the grid-aligned bounding box area around the shape provided as the filter and M is the number of items inside the shape", - "Arguments": [ - { - "TypeDiscriminator": "RespCommandKeyArgument", - "Name": "DESTINATION", - "DisplayText": "destination", - "Type": "Key", - "KeySpecIndex": 0 - }, - { - "TypeDiscriminator": "RespCommandKeyArgument", - "Name": "SOURCE", - "DisplayText": "source", - "Type": "Key", - "KeySpecIndex": 1 - }, - { - "TypeDiscriminator": "RespCommandContainerArgument", - "Name": "FROM", - "Type": "OneOf", - "Arguments": [ - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "MEMBER", - "DisplayText": "member", - "Type": "String", - "Token": "FROMMEMBER" - }, - { - "TypeDiscriminator": "RespCommandContainerArgument", - "Name": "FROMLONLAT", - "Type": "Block", - "Token": "FROMLONLAT", - "Arguments": [ - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "LONGITUDE", - "DisplayText": "longitude", - "Type": "Double" - }, - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "LATITUDE", - "DisplayText": "latitude", - "Type": "Double" - } - ] - } - ] - }, - { - "TypeDiscriminator": "RespCommandContainerArgument", - "Name": "BY", - "Type": "OneOf", - "Arguments": [ - { - "TypeDiscriminator": "RespCommandContainerArgument", - "Name": "CIRCLE", - "Type": "Block", - "Arguments": [ - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "RADIUS", - "DisplayText": "radius", - "Type": "Double", - "Token": "BYRADIUS" - }, - { - "TypeDiscriminator": "RespCommandContainerArgument", - "Name": "UNIT", - "Type": "OneOf", - "Arguments": [ - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "M", - "DisplayText": "m", - "Type": "PureToken", - "Token": "M" - }, - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "KM", - "DisplayText": "km", - "Type": "PureToken", - "Token": "KM" - }, - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "FT", - "DisplayText": "ft", - "Type": "PureToken", - "Token": "FT" - }, - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "MI", - "DisplayText": "mi", - "Type": "PureToken", - "Token": "MI" - } - ] - } - ] - }, - { - "TypeDiscriminator": "RespCommandContainerArgument", - "Name": "BOX", - "Type": "Block", - "Arguments": [ - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "WIDTH", - "DisplayText": "width", - "Type": "Double", - "Token": "BYBOX" - }, - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "HEIGHT", - "DisplayText": "height", - "Type": "Double" - }, - { - "TypeDiscriminator": "RespCommandContainerArgument", - "Name": "UNIT", - "Type": "OneOf", - "Arguments": [ - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "M", - "DisplayText": "m", - "Type": "PureToken", - "Token": "M" - }, - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "KM", - "DisplayText": "km", - "Type": "PureToken", - "Token": "KM" - }, - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "FT", - "DisplayText": "ft", - "Type": "PureToken", - "Token": "FT" - }, - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "MI", - "DisplayText": "mi", - "Type": "PureToken", - "Token": "MI" - } - ] - } - ] - } - ] - }, - { - "TypeDiscriminator": "RespCommandContainerArgument", - "Name": "ORDER", - "Type": "OneOf", - "ArgumentFlags": "Optional", - "Arguments": [ - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "ASC", - "DisplayText": "asc", - "Type": "PureToken", - "Token": "ASC" - }, - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "DESC", - "DisplayText": "desc", - "Type": "PureToken", - "Token": "DESC" - } - ] - }, - { - "TypeDiscriminator": "RespCommandContainerArgument", - "Name": "COUNT-BLOCK", - "Type": "Block", - "ArgumentFlags": "Optional", - "Arguments": [ - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "COUNT", - "DisplayText": "count", - "Type": "Integer", - "Token": "COUNT" - }, - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "ANY", - "DisplayText": "any", - "Type": "PureToken", - "Token": "ANY", - "ArgumentFlags": "Optional" - } - ] - }, - { - "TypeDiscriminator": "RespCommandBasicArgument", - "Name": "STOREDIST", - "DisplayText": "storedist", - "Type": "PureToken", - "Token": "STOREDIST", - "ArgumentFlags": "Optional" - } - ] - }, { "Command": "GET", "Name": "GET", diff --git a/libs/resources/RespCommandsInfo.json b/libs/resources/RespCommandsInfo.json index 258561bb5a..9ff0a95864 100644 --- a/libs/resources/RespCommandsInfo.json +++ b/libs/resources/RespCommandsInfo.json @@ -1276,44 +1276,6 @@ } ] }, - { - "Command": "GEOSEARCHSTORE", - "Name": "GEOSEARCHSTORE", - "Arity": -8, - "Flags": "DenyOom, Write", - "FirstKey": 1, - "LastKey": 2, - "Step": 1, - "AclCategories": "Geo, Slow, Write", - "KeySpecifications": [ - { - "BeginSearch": { - "TypeDiscriminator": "BeginSearchIndex", - "Index": 1 - }, - "FindKeys": { - "TypeDiscriminator": "FindKeysRange", - "LastKey": 0, - "KeyStep": 1, - "Limit": 0 - }, - "Flags": "OW, Update" - }, - { - "BeginSearch": { - "TypeDiscriminator": "BeginSearchIndex", - "Index": 2 - }, - "FindKeys": { - "TypeDiscriminator": "FindKeysRange", - "LastKey": 0, - "KeyStep": 1, - "Limit": 0 - }, - "Flags": "RO, Access" - } - ] - }, { "Command": "GET", "Name": "GET", diff --git a/libs/server/API/GarnetApiObjectCommands.cs b/libs/server/API/GarnetApiObjectCommands.cs index 3690e93599..e045363cb3 100644 --- a/libs/server/API/GarnetApiObjectCommands.cs +++ b/libs/server/API/GarnetApiObjectCommands.cs @@ -145,10 +145,6 @@ public GarnetStatus GeoAdd(byte[] key, ref ObjectInput input, ref GarnetObjectSt public GarnetStatus GeoCommands(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter) => storageSession.GeoCommands(key, ref input, ref outputFooter, ref objectContext); - /// - public GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref ObjectInput input, ref SpanByteAndMemory output) - => storageSession.GeoSearchStore(key, destinationKey, ref input, ref output, ref objectContext); - #endregion #region List Methods diff --git a/libs/server/API/IGarnetApi.cs b/libs/server/API/IGarnetApi.cs index 2a4ca7a3fd..e271407c63 100644 --- a/libs/server/API/IGarnetApi.cs +++ b/libs/server/API/IGarnetApi.cs @@ -482,15 +482,6 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// GarnetStatus GeoAdd(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter); - /// - /// Geospatial search and store in destination key. - /// - /// - /// - /// - /// - GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destinationKey, ref ObjectInput input, ref SpanByteAndMemory output); - #endregion #region Set Methods diff --git a/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs b/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs index ff0526721e..feee1c7e8e 100644 --- a/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs +++ b/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs @@ -437,14 +437,6 @@ private void GeoSearch(ref ObjectInput input, ref SpanByteAndMemory output) return; } - // Not supported options in Garnet: WITHHASH - if (opts.WithHash) - { - while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_GENERIC_UNK_CMD, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - return; - } - // Get the results // FROMMEMBER if (opts.FromMember && sortedSetDict.TryGetValue(fromMember, out var centerPointScore)) @@ -483,64 +475,30 @@ private void GeoSearch(ref ObjectInput input, ref SpanByteAndMemory output) } } - if (responseData.Count == 0) - { - while (!RespWriteUtils.WriteInteger(0, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - } - else - { - var innerArrayLength = 1; - if (opts.WithDist) - { - innerArrayLength++; - } - if (opts.WithHash) - { - innerArrayLength++; - } - if (opts.WithCoord) - { - innerArrayLength++; - } + // Write results + while (!RespWriteUtils.WriteArrayLength(responseData.Count, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - // Write results - while (!RespWriteUtils.WriteArrayLength(responseData.Count, ref curr, end)) + foreach (var item in responseData) + { + while (!RespWriteUtils.WriteBulkString(item.Member, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - foreach (var item in responseData) - { - if (innerArrayLength > 1) - { - while (!RespWriteUtils.WriteArrayLength(innerArrayLength, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - } - - while (!RespWriteUtils.WriteBulkString(item.Member, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - - if (opts.WithDist) - { - var distanceValue = (byBoxUnits.Length == 1 && (byBoxUnits[0] == (int)'M' || byBoxUnits[0] == (int)'m')) ? item.Distance - : server.GeoHash.ConvertMetersToUnits(item.Distance, byBoxUnits); + var distanceValue = (byBoxUnits.Length == 1 && (byBoxUnits[0] == (int)'M' || byBoxUnits[0] == (int)'m')) ? item.Distance + : server.GeoHash.ConvertMetersToUnits(item.Distance, byBoxUnits); - while (!RespWriteUtils.TryWriteDoubleBulkString(distanceValue, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - } + while (!RespWriteUtils.TryWriteDoubleBulkString(distanceValue, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - if (opts.WithCoord) - { - // Write array of 2 values - while (!RespWriteUtils.WriteArrayLength(2, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + // Write array of 2 values + while (!RespWriteUtils.WriteArrayLength(2, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - while (!RespWriteUtils.TryWriteDoubleBulkString(item.Coordinates.Longitude, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + while (!RespWriteUtils.TryWriteDoubleBulkString(item.Coordinates.Longitude, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - while (!RespWriteUtils.TryWriteDoubleBulkString(item.Coordinates.Latitude, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - } - } + while (!RespWriteUtils.TryWriteDoubleBulkString(item.Coordinates.Latitude, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); } } } diff --git a/libs/server/Resp/CmdStrings.cs b/libs/server/Resp/CmdStrings.cs index 87e429bc33..4815c1c8e9 100644 --- a/libs/server/Resp/CmdStrings.cs +++ b/libs/server/Resp/CmdStrings.cs @@ -104,9 +104,6 @@ static partial class CmdStrings public static ReadOnlySpan rank => "rank"u8; public static ReadOnlySpan MAXLEN => "MAXLEN"u8; public static ReadOnlySpan maxlen => "maxlen"u8; - public static ReadOnlySpan STOREDIST => "STOREDIST"u8; - public static ReadOnlySpan WITHDIST => "WITHDIST"u8; - public static ReadOnlySpan WITHHASH => "WITHHASH"u8; /// /// Response strings diff --git a/libs/server/Resp/Objects/SortedSetGeoCommands.cs b/libs/server/Resp/Objects/SortedSetGeoCommands.cs index abf9dbdccd..1d36fa6001 100644 --- a/libs/server/Resp/Objects/SortedSetGeoCommands.cs +++ b/libs/server/Resp/Objects/SortedSetGeoCommands.cs @@ -143,47 +143,5 @@ private unsafe bool GeoCommands(RespCommand command, ref TGarnetApi return true; } - - /// - /// GEOSEARCHSTORE: Store the the members of a sorted set populated with geospatial data, which are within the borders of the area specified by a given shape. - /// - /// - /// - /// - private unsafe bool GeoSearchStore(ref TGarnetApi storageApi) - where TGarnetApi : IGarnetApi - { - if (parseState.Count < 4) - { - return AbortWithWrongNumberOfArguments(nameof(RespCommand.GEOSEARCHSTORE)); - } - - var destinationKey = parseState.GetArgSliceByRef(0); - var sourceKey = parseState.GetArgSliceByRef(1); - - var input = new ObjectInput(new RespInputHeader - { - type = GarnetObjectType.SortedSet, - }, ref parseState, 2); - - var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); - var status = storageApi.GeoSearchStore(sourceKey, destinationKey, ref input, ref output); - - switch (status) - { - case GarnetStatus.OK: - if (!output.IsSpanByte) - SendAndReset(output.Memory, output.Length); - else - dcurr += output.Length; - break; - case GarnetStatus.WRONGTYPE: - while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) - SendAndReset(); - break; - } - - return true; - } } } \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index 5b13c43fec..7cd956ebb3 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -97,7 +97,6 @@ public enum RespCommand : ushort FLUSHALL, FLUSHDB, GEOADD, - GEOSEARCHSTORE, GETDEL, GETEX, GETSET, @@ -1416,10 +1415,6 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.ZREMRANGEBYLEX; } - else if (*(ulong*)(ptr + 3) == MemoryMarshal.Read("\r\nGEOSEA"u8) && *(ulong*)(ptr + 11) == MemoryMarshal.Read("RCHSTORE"u8) && *(ushort*)(ptr + 19) == MemoryMarshal.Read("\r\n"u8)) - { - return RespCommand.GEOSEARCHSTORE; - } break; case 15: diff --git a/libs/server/Resp/Parser/SessionParseState.cs b/libs/server/Resp/Parser/SessionParseState.cs index 3975062d30..ee916fc860 100644 --- a/libs/server/Resp/Parser/SessionParseState.cs +++ b/libs/server/Resp/Parser/SessionParseState.cs @@ -162,21 +162,6 @@ public void InitializeWithArguments(ArgSlice[] args) } } - /// - /// Initialize the parse state with a given set of arguments - /// - /// Set of arguments to initialize buffer with - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public void InitializeWithArguments(Span args) - { - Initialize(args.Length); - - for (var i = 0; i < args.Length; i++) - { - buffer[i] = args[i]; - } - } - /// /// Set argument at a specific index /// diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index 426120dc61..ef7d28d8ac 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -611,7 +611,6 @@ private bool ProcessArrayCommands(RespCommand cmd, ref TGarnetApi st RespCommand.GEODIST => GeoCommands(cmd, ref storageApi), RespCommand.GEOPOS => GeoCommands(cmd, ref storageApi), RespCommand.GEOSEARCH => GeoCommands(cmd, ref storageApi), - RespCommand.GEOSEARCHSTORE => GeoSearchStore(ref storageApi), //HLL Commands RespCommand.PFADD => HyperLogLogAdd(ref storageApi), RespCommand.PFMERGE => HyperLogLogMerge(ref storageApi), diff --git a/libs/server/Storage/Session/ObjectStore/SortedSetGeoOps.cs b/libs/server/Storage/Session/ObjectStore/SortedSetGeoOps.cs index 479745c2b1..00a3edf742 100644 --- a/libs/server/Storage/Session/ObjectStore/SortedSetGeoOps.cs +++ b/libs/server/Storage/Session/ObjectStore/SortedSetGeoOps.cs @@ -2,9 +2,6 @@ // Licensed under the MIT license. using System; -using System.Buffers; -using System.Diagnostics; -using Garnet.common; using Tsavorite.core; namespace Garnet.server @@ -44,160 +41,5 @@ public GarnetStatus GeoCommands(byte[] key, ref ObjectInput inpu where TObjectContext : ITsavoriteContext => ReadObjectStoreOperationWithOutput(key, ref input, ref objectContext, ref outputFooter); - /// - /// Geospatial search and store in destination key. - /// - /// - /// - /// - /// - /// - /// - public unsafe GarnetStatus GeoSearchStore(ArgSlice key, ArgSlice destination, ref ObjectInput input, ref SpanByteAndMemory output, ref TObjectContext objectContext) - where TObjectContext : ITsavoriteContext - { - var createTransaction = false; - - if (txnManager.state != TxnState.Running) - { - Debug.Assert(txnManager.state == TxnState.None); - createTransaction = true; - txnManager.SaveKeyEntryToLock(destination, true, LockType.Exclusive); - txnManager.SaveKeyEntryToLock(key, true, LockType.Shared); - _ = txnManager.Run(true); - } - var objectStoreLockableContext = txnManager.ObjectStoreLockableContext; - - var isMemory = false; - MemoryHandle ptrHandle = default; - var ptr = output.SpanByte.ToPointer(); - var curr = ptr; - var end = curr + output.Length; - - try - { - var isStoreDist = false; - Span geoSearchParseState = stackalloc ArgSlice[input.parseState.Count - input.parseStateFirstArgIdx + 1]; - var currArgIdx = 0; - var i = input.parseStateFirstArgIdx; - while (i < input.parseState.Count) - { - if (!isStoreDist && input.parseState.GetArgSliceByRef(i).ReadOnlySpan.EqualsUpperCaseSpanIgnoringCase(CmdStrings.STOREDIST)) - { - isStoreDist = true; - break; - } - else - { - geoSearchParseState[currArgIdx] = input.parseState.GetArgSliceByRef(i); - currArgIdx++; - } - i++; - } - geoSearchParseState[currArgIdx++] = isStoreDist ? ArgSlice.FromPinnedSpan(CmdStrings.WITHDIST) : ArgSlice.FromPinnedSpan(CmdStrings.WITHHASH); - - var sourceKey = key.ToArray(); - var parseState = new SessionParseState(); - parseState.InitializeWithArguments(geoSearchParseState.Slice(0, currArgIdx)); - - var searchInput = new ObjectInput(new RespInputHeader - { - type = GarnetObjectType.SortedSet, - SortedSetOp = SortedSetOperation.GEOSEARCH, - }, ref parseState, 0); - - SpanByteAndMemory searchOutMem = default; - var searchOut = new GarnetObjectStoreOutput { spanByteAndMemory = searchOutMem }; - var status = GeoCommands(sourceKey, ref searchInput, ref searchOut, ref objectStoreLockableContext); - searchOutMem = searchOut.spanByteAndMemory; - - if (status == GarnetStatus.WRONGTYPE) - { - return GarnetStatus.WRONGTYPE; - } - - if (status == GarnetStatus.NOTFOUND) - { - _ = EXPIRE(destination, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, ref lockableContext, ref objectStoreLockableContext); - while (!RespWriteUtils.WriteInteger(0, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - return GarnetStatus.OK; - } - - Debug.Assert(!searchOutMem.IsSpanByte, "Output should not be in SpanByte format when the status is OK"); - - var searchOutHandler = searchOutMem.Memory.Memory.Pin(); - try - { - - var searchOutPtr = (byte*)searchOutHandler.Pointer; - var currOutPtr = searchOutPtr; - var endOutPtr = searchOutPtr + searchOutMem.Length; - - if (RespReadUtils.TryReadErrorAsSpan(out var error, ref currOutPtr, endOutPtr)) - { - while (!RespWriteUtils.WriteError(error, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - return GarnetStatus.OK; - } - - var destinationKey = destination.ToArray(); - objectStoreLockableContext.Delete(ref destinationKey); - - RespReadUtils.ReadUnsignedArrayLength(out var foundItems, ref currOutPtr, endOutPtr); - - // Prepare the parse state for sorted set add - var zParseState = new SessionParseState(); - zParseState.Initialize(foundItems * 2); - - for (int j = 0; j < foundItems; j++) - { - RespReadUtils.ReadUnsignedArrayLength(out var innerLength, ref currOutPtr, endOutPtr); - Debug.Assert(innerLength == 2, "Should always has location and hash or distance"); - - RespReadUtils.TrySliceWithLengthHeader(out var location, ref currOutPtr, endOutPtr); - if (isStoreDist) - { - RespReadUtils.ReadSpanWithLengthHeader(out var score, ref currOutPtr, endOutPtr); - zParseState.SetArgument(2 * j, ArgSlice.FromPinnedSpan(score)); - zParseState.SetArgument((2 * j) + 1, ArgSlice.FromPinnedSpan(location)); - } - else - { - RespReadUtils.ReadIntegerAsSpan(out var score, ref currOutPtr, endOutPtr); - zParseState.SetArgument(2 * j, ArgSlice.FromPinnedSpan(score)); - zParseState.SetArgument((2 * j) + 1, ArgSlice.FromPinnedSpan(location)); - } - } - - // Prepare the input - var zAddInput = new ObjectInput(new RespInputHeader - { - type = GarnetObjectType.SortedSet, - SortedSetOp = SortedSetOperation.ZADD, - }, ref zParseState, 0); - - var zAddOutput = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(null) }; - RMWObjectStoreOperationWithOutput(destinationKey, ref zAddInput, ref objectStoreLockableContext, ref zAddOutput); - - while (!RespWriteUtils.WriteInteger(foundItems, ref curr, end)) - ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - } - finally - { - searchOutHandler.Dispose(); - } - - return GarnetStatus.OK; - } - finally - { - if (createTransaction) - txnManager.Commit(true); - - if (isMemory) ptrHandle.Dispose(); - output.Length = (int)(curr - ptr); - } - } } } \ No newline at end of file diff --git a/playground/CommandInfoUpdater/SupportedCommand.cs b/playground/CommandInfoUpdater/SupportedCommand.cs index ca140e91b0..5ccf20af5e 100644 --- a/playground/CommandInfoUpdater/SupportedCommand.cs +++ b/playground/CommandInfoUpdater/SupportedCommand.cs @@ -124,7 +124,6 @@ public class SupportedCommand new("GEOHASH", RespCommand.GEOHASH), new("GEOPOS", RespCommand.GEOPOS), new("GEOSEARCH", RespCommand.GEOSEARCH), - new("GEOSEARCHSTORE", RespCommand.GEOSEARCHSTORE), new("GET", RespCommand.GET), new("GETEX", RespCommand.GETEX), new("GETBIT", RespCommand.GETBIT), diff --git a/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs b/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs index 3c3bd7ed4a..dcd12f1f6a 100644 --- a/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs +++ b/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs @@ -347,32 +347,6 @@ public override string[] GetSingleSlotRequest() public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); } - internal class GEOSEARCHSTORE : BaseCommand - { - public override bool IsArrayCommand => false; - public override bool ArrayResponse => false; - public override string Command => nameof(GEOSEARCHSTORE); - - public override string[] GetSingleSlotRequest() - { - var ssk = GetSingleSlotKeys; - return [ssk[0], ssk[1], "FROMMEMBER", "bar", "BYBOX", "800", "800", "km", "STOREDIST"]; - } - - public override string[] GetCrossSlotRequest() - { - var csk = GetCrossSlotKeys; - return [csk[0], csk[1], "FROMMEMBER", "bar", "BYBOX", "800", "800", "km", "STOREDIST"]; - } - - public override ArraySegment[] SetupSingleSlotRequest() - { - var ssk = GetSingleSlotKeys; - var setup = new ArraySegment[] { new ArraySegment([ssk[0], ssk[1], "FROMMEMBER", "bar", "BYBOX", "800", "800", "km", "STOREDIST"]) }; - return setup; - } - } - internal class SETRANGE : BaseCommand { public override bool IsArrayCommand => false; diff --git a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs b/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs index 2e75f29dad..429eb6543e 100644 --- a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs +++ b/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs @@ -90,7 +90,6 @@ public class ClusterSlotVerificationTests new SRANDMEMBER(), new GEOADD(), new GEOHASH(), - new GEOSEARCHSTORE(), new ZADD(), new ZREM(), new ZCARD(), @@ -268,7 +267,6 @@ public virtual void OneTimeTearDown() [TestCase("SRANDMEMBER")] [TestCase("GEOADD")] [TestCase("GEOHASH")] - [TestCase("GEOSEARCHSTORE")] [TestCase("ZADD")] [TestCase("ZREM")] [TestCase("ZCARD")] @@ -406,7 +404,6 @@ void GarnetClientSessionClusterDown(BaseCommand command) [TestCase("SRANDMEMBER")] [TestCase("GEOADD")] [TestCase("GEOHASH")] - [TestCase("GEOSEARCHSTORE")] [TestCase("ZADD")] [TestCase("ZREM")] [TestCase("ZCARD")] @@ -554,7 +551,6 @@ void GarnetClientSessionOK(BaseCommand command) [TestCase("SRANDMEMBER")] [TestCase("GEOADD")] [TestCase("GEOHASH")] - [TestCase("GEOSEARCHSTORE")] [TestCase("ZADD")] [TestCase("ZREM")] [TestCase("ZCARD")] @@ -694,7 +690,6 @@ void GarnetClientSessionCrossslotTest(BaseCommand command) [TestCase("SRANDMEMBER")] [TestCase("GEOADD")] [TestCase("GEOHASH")] - [TestCase("GEOSEARCHSTORE")] [TestCase("ZADD")] [TestCase("ZREM")] [TestCase("ZCARD")] @@ -841,7 +836,6 @@ void GarnetClientSessionMOVEDTest(BaseCommand command) [TestCase("SRANDMEMBER")] [TestCase("GEOADD")] [TestCase("GEOHASH")] - [TestCase("GEOSEARCHSTORE")] [TestCase("ZADD")] [TestCase("ZREM")] [TestCase("ZCARD")] @@ -1005,7 +999,6 @@ void GarnetClientSessionASKTest(BaseCommand command) [TestCase("SRANDMEMBER")] [TestCase("GEOADD")] [TestCase("GEOHASH")] - [TestCase("GEOSEARCHSTORE")] [TestCase("ZADD")] [TestCase("ZREM")] [TestCase("ZCARD")] diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/Garnet.test/Resp/ACL/RespCommandTests.cs index 4f0f684219..c71932497f 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/Garnet.test/Resp/ACL/RespCommandTests.cs @@ -5480,22 +5480,6 @@ static async Task DoGeoSearchAsync(GarnetClient client) } } - [Test] - public async Task GeoSearchStoreACLsAsync() - { - await CheckCommandsAsync( - "GEOSEARCHSTORE", - [DoGeoSearchStoreAsync], - skipPermitted: true - ); - - static async Task DoGeoSearchStoreAsync(GarnetClient client) - { - var val = await client.ExecuteForLongResultAsync("GEOSEARCHSTORE", ["bar", "foo", "FROMMEMBER", "bar", "BYBOX", "2", "2", "M", "STOREDIST"]); - ClassicAssert.AreEqual(0, val); - } - } - [Test] public async Task ZAddACLsAsync() { diff --git a/test/Garnet.test/RespSortedSetGeoTests.cs b/test/Garnet.test/RespSortedSetGeoTests.cs index b7c6d1cc88..3c855a50e0 100644 --- a/test/Garnet.test/RespSortedSetGeoTests.cs +++ b/test/Garnet.test/RespSortedSetGeoTests.cs @@ -274,128 +274,6 @@ public void CheckGeoSortedSetOperationsOnWrongTypeObjectSE() RespTestsUtils.CheckCommandOnWrongTypeObjectSE(() => db.GeoSearch(keys[0], values[0][1], new GeoSearchBox(800, 800, GeoUnit.Kilometers))); } - [Test] - public void CanUseGeoSearch() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - var entries = new GeoEntry[cities.GetLength(0)]; - var key = new RedisKey("cities"); - var destinationKey = new RedisKey("newCities"); - for (int j = 0; j < cities.GetLength(0); j++) - { - entries[j] = new GeoEntry( - double.Parse(cities[j, 0], CultureInfo.InvariantCulture), - double.Parse(cities[j, 1], CultureInfo.InvariantCulture), - new RedisValue(cities[j, 2])); - } - var response = db.GeoAdd(key, entries, CommandFlags.None); - - var res = db.GeoSearch(key, new RedisValue("Washington"), new GeoSearchBox(800, 800, GeoUnit.Kilometers), options: GeoRadiusOptions.None); - ClassicAssert.AreEqual(3, res.Length); - ClassicAssert.AreEqual("Washington", (string)res[0].Member); - ClassicAssert.AreEqual(res[0].Distance, null); - ClassicAssert.AreEqual(res[0].Position, null); - ClassicAssert.AreEqual("Philadelphia", (string)res[1].Member); - ClassicAssert.AreEqual(res[1].Distance, null); - ClassicAssert.AreEqual(res[1].Position, null); - ClassicAssert.AreEqual("New York", (string)res[2].Member); - ClassicAssert.AreEqual(res[2].Distance, null); - ClassicAssert.AreEqual(res[2].Position, null); - - res = db.GeoSearch(key, new RedisValue("Washington"), new GeoSearchBox(800, 800, GeoUnit.Kilometers), options: GeoRadiusOptions.WithDistance); - ClassicAssert.AreEqual(3, res.Length); - ClassicAssert.AreEqual("Washington", (string)res[0].Member); - Assert.That(res[0].Distance, Is.EqualTo(0).Within(1.0 / Math.Pow(10, 6))); - ClassicAssert.AreEqual(res[0].Position, null); - ClassicAssert.AreEqual("Philadelphia", (string)res[1].Member); - Assert.That(res[1].Distance, Is.EqualTo(198.424300439725).Within(1.0 / Math.Pow(10, 6))); - ClassicAssert.AreEqual(res[1].Position, null); - ClassicAssert.AreEqual("New York", (string)res[2].Member); - Assert.That(res[2].Distance, Is.EqualTo(327.676458633557).Within(1.0 / Math.Pow(10, 6))); - ClassicAssert.AreEqual(res[2].Position, null); - - res = db.GeoSearch(key, new RedisValue("Washington"), new GeoSearchBox(800, 800, GeoUnit.Kilometers), options: GeoRadiusOptions.WithCoordinates); - ClassicAssert.AreEqual(3, res.Length); - ClassicAssert.AreEqual("Washington", (string)res[0].Member); - ClassicAssert.AreEqual(res[0].Distance, null); - Assert.That(res[0].Position.Value.Longitude, Is.EqualTo(-77.03687042).Within(1.0 / Math.Pow(10, 6))); - Assert.That(res[0].Position.Value.Latitude, Is.EqualTo(38.9071919).Within(1.0 / Math.Pow(10, 6))); - ClassicAssert.AreEqual("Philadelphia", (string)res[1].Member); - ClassicAssert.AreEqual(res[1].Distance, null); - Assert.That(res[1].Position.Value.Longitude, Is.EqualTo(-75.1652196).Within(1.0 / Math.Pow(10, 6))); - Assert.That(res[1].Position.Value.Latitude, Is.EqualTo(39.95258287).Within(1.0 / Math.Pow(10, 6))); - ClassicAssert.AreEqual("New York", (string)res[2].Member); - ClassicAssert.AreEqual(res[2].Distance, null); - Assert.That(res[2].Position.Value.Longitude, Is.EqualTo(-74.00594205).Within(1.0 / Math.Pow(10, 6))); - Assert.That(res[2].Position.Value.Latitude, Is.EqualTo(40.71278259).Within(1.0 / Math.Pow(10, 6))); - - res = db.GeoSearch(key, new RedisValue("Washington"), new GeoSearchBox(800, 800, GeoUnit.Kilometers), options: GeoRadiusOptions.WithDistance | GeoRadiusOptions.WithCoordinates); - ClassicAssert.AreEqual(3, res.Length); - ClassicAssert.AreEqual("Washington", (string)res[0].Member); - Assert.That(res[0].Distance, Is.EqualTo(0).Within(1.0 / Math.Pow(10, 6))); - Assert.That(res[0].Position.Value.Longitude, Is.EqualTo(-77.03687042).Within(1.0 / Math.Pow(10, 6))); - Assert.That(res[0].Position.Value.Latitude, Is.EqualTo(38.9071919).Within(1.0 / Math.Pow(10, 6))); - ClassicAssert.AreEqual("Philadelphia", (string)res[1].Member); - Assert.That(res[1].Distance, Is.EqualTo(198.424300439725).Within(1.0 / Math.Pow(10, 6))); - Assert.That(res[1].Position.Value.Longitude, Is.EqualTo(-75.1652196).Within(1.0 / Math.Pow(10, 6))); - Assert.That(res[1].Position.Value.Latitude, Is.EqualTo(39.95258287).Within(1.0 / Math.Pow(10, 6))); - ClassicAssert.AreEqual("New York", (string)res[2].Member); - Assert.That(res[2].Distance, Is.EqualTo(327.676458633557).Within(1.0 / Math.Pow(10, 6))); - Assert.That(res[2].Position.Value.Longitude, Is.EqualTo(-74.00594205).Within(1.0 / Math.Pow(10, 6))); - Assert.That(res[2].Position.Value.Latitude, Is.EqualTo(40.71278259).Within(1.0 / Math.Pow(10, 6))); - } - - [Test] - public void CanUseGeoSearchStore() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - var entries = new GeoEntry[cities.GetLength(0)]; - var key = new RedisKey("cities"); - var destinationKey = new RedisKey("newCities"); - for (int j = 0; j < cities.GetLength(0); j++) - { - entries[j] = new GeoEntry( - double.Parse(cities[j, 0], CultureInfo.InvariantCulture), - double.Parse(cities[j, 1], CultureInfo.InvariantCulture), - new RedisValue(cities[j, 2])); - } - db.GeoAdd(key, entries, CommandFlags.None); - - db.SortedSetAdd(destinationKey, "OldValue", 10); // Add a value to be replaced - - var actualCount = db.GeoSearchAndStore(key, destinationKey, new RedisValue("Washington"), new GeoSearchBox(800, 800, GeoUnit.Kilometers), storeDistances: true); - ClassicAssert.AreEqual(3, actualCount); - - var actualValues = db.SortedSetRangeByScoreWithScores(destinationKey); - ClassicAssert.AreEqual(3, actualValues.Length); - ClassicAssert.AreEqual("Washington", (string)actualValues[0].Element); - Assert.That(actualValues[0].Score, Is.EqualTo(0).Within(1.0 / Math.Pow(10, 6))); - ClassicAssert.AreEqual("Philadelphia", (string)actualValues[1].Element); - Assert.That(actualValues[1].Score, Is.EqualTo(198.424300439725).Within(1.0 / Math.Pow(10, 6))); - ClassicAssert.AreEqual("New York", (string)actualValues[2].Element); - Assert.That(actualValues[2].Score, Is.EqualTo(327.676458633557).Within(1.0 / Math.Pow(10, 6))); - } - - [Test] - public void CanUseGeoSearchStoreWithDeleteKeyWhenSourceNotFound() - { - using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); - var db = redis.GetDatabase(0); - var entries = new GeoEntry[cities.GetLength(0)]; - var key = new RedisKey("cities"); - var destinationKey = new RedisKey("newCities"); - - db.SortedSetAdd(destinationKey, "OldValue", 10); - - var actualCount = db.GeoSearchAndStore(key, destinationKey, new RedisValue("Washington"), new GeoSearchBox(800, 800, GeoUnit.Kilometers), storeDistances: true); - ClassicAssert.AreEqual(0, actualCount); - - var actualValues = db.SortedSetRangeByScoreWithScores(destinationKey); - ClassicAssert.AreEqual(0, actualValues.Length); - } - //end region of SE tests #endregion @@ -423,14 +301,14 @@ public void CanUseGeoSearchWithCities(int bytesSent) //TODO: Assert values for latitude and longitude //TODO: Review precision to use for all framework versions using var lightClientRequest = TestUtils.CreateRequest(); - var responseBuf = lightClientRequest.SendCommands("GEOSEARCH cities FROMMEMBER Washington BYBOX 800 800 km WITHCOORD WITHDIST", "PING"); - var expectedResponse = "*3\r\n*3\r\n$10\r\nWashington\r\n$1\r\n0\r\n*2\r\n$17\r\n-77.0368704199791\r\n$17\r\n38.90719190239906\r\n*3\r\n$12\r\nPhiladelphia\r\n$17\r\n198.4242996738795\r\n*2\r\n$18\r\n-75.16521960496902\r\n$18\r\n39.952582865953445\r\n*3\r\n$8\r\nNew York\r\n$18\r\n327.67645879712575\r\n*2\r\n$17\r\n-74.0059420466423\r\n$18\r\n40.712782591581345\r\n+PONG\r\n"; + var responseBuf = lightClientRequest.SendCommands("GEOSEARCH cities FROMMEMBER Washington BYBOX 800 800 km WITHCOORD WITHDIST WITHHASH", "PING", 16); + var expectedResponse = "*3\r\n$10\r\nWashington\r\n$1\r\n0\r\n*2\r\n$12\r\n-77.03687042\r\n$10\r\n38.9071919\r\n$12\r\nPhiladelphia\r\n$16\r\n198.424300439725\r\n*2\r\n$11\r\n-75.1652196\r\n$11\r\n39.95258287\r\n$8\r\nNew York\r\n$16\r\n327.676458633557\r\n*2\r\n$12\r\n-74.00594205\r\n$11\r\n40.71278259\r\n+PONG\r\n"; var actualValue = Encoding.ASCII.GetString(responseBuf).Substring(0, expectedResponse.Length); ClassicAssert.IsTrue(actualValue.IndexOf("Washington") != -1); //Send command in chunks - responseBuf = lightClientRequest.SendCommandChunks("GEOSEARCH cities FROMMEMBER Washington BYBOX 800 800 km COUNT 3 ANY WITHCOORD WITHDIST", bytesSent, 16); - expectedResponse = "*3\r\n*3\r\n$10\r\nWashington\r\n$1\r\n0\r\n*2\r\n$17\r\n-77.0368704199791\r\n$17\r\n38.90719190239906\r\n*3\r\n$12\r\nPhiladelphia\r\n$17\r\n198.4242996738795\r\n*2\r\n$18\r\n-75.16521960496902\r\n$18\r\n39.952582865953445\r\n*3\r\n$8\r\nNew York\r\n$18\r\n327.67645879712575\r\n*2\r\n$17\r\n-74.0059420466423\r\n$18\r\n40.712782591581345\r\n+PONG\r\n"; + responseBuf = lightClientRequest.SendCommandChunks("GEOSEARCH cities FROMMEMBER Washington BYBOX 800 800 km COUNT 3 ANY WITHCOORD WITHDIST WITHHASH", bytesSent, 16); + expectedResponse = "*3\r\n$10\r\nWashington\r\n$1\r\n0\r\n*2\r\n$12\r\n-77.03687042\r\n$10\r\n38.9071919\r\n$12\r\nPhiladelphia\r\n$16\r\n198.424300439725\r\n*2\r\n$11\r\n-75.1652196\r\n$11\r\n39.95258287\r\n$8\r\nNew York\r\n$16\r\n327.676458633557\r\n*2\r\n$12\r\n-74.00594205\r\n$11\r\n40.71278259\r\n+PONG\r\n"; actualValue = Encoding.ASCII.GetString(responseBuf).Substring(0, expectedResponse.Length); ClassicAssert.IsTrue(actualValue.IndexOf("Washington") != -1); } diff --git a/website/docs/commands/api-compatibility.md b/website/docs/commands/api-compatibility.md index 8d694d3ffa..fa6b2bb6d8 100644 --- a/website/docs/commands/api-compatibility.md +++ b/website/docs/commands/api-compatibility.md @@ -160,7 +160,7 @@ Note that this list is subject to change as we continue to expand our API comman | | GEORADIUSBYMEMBER | ➖ | (Deprecated) | | | GEORADIUSBYMEMBER_RO | ➖ | (Deprecated) | | | [GEOSEARCH](data-structures.md#geosearch) | ➕ | Partially Implemented | -| | [GEOSEARCHSTORE](data-structures.md#geosearchstore) | ➕ | Partially Implemented | +| | GEOSEARCHSTORE | ➖ | | | **HASH** | [HDEL](data-structures.md#hdel) | ➕ | | | | [HEXISTS](data-structures.md#hexists) | ➕ | | | | HEXPIRE | ➖ | | diff --git a/website/docs/commands/data-structures.md b/website/docs/commands/data-structures.md index 0d4dad004a..7c5a4d4a4c 100644 --- a/website/docs/commands/data-structures.md +++ b/website/docs/commands/data-structures.md @@ -1174,22 +1174,3 @@ An Array reply of matched members, where each sub-array represents a single item --- -### GEOSEARCHSTORE - -#### Syntax - -```bash -GEOSEARCHSTORE destination source - | BYBOX width height > [ASC | DESC] [COUNT count - [ANY]] [STOREDIST] -``` - -This command is like [GEOSEARCH](#geosearch), but stores the result in destination key. - -**Reply** - -Integer reply: the number of elements in the resulting set - ---- - From c580f2253ed1987323e8e1791dcdef16b8942979 Mon Sep 17 00:00:00 2001 From: Yoganand Rajasekaran <60369795+yrajas@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:03:21 -0700 Subject: [PATCH 11/15] BDN benchmark for custom operations. (#750) * Initial commit * Custom Operations. * Formatting, comments updated. --- benchmark/BDN.benchmark/BDN.benchmark.csproj | 9 +- .../BDN.benchmark/Cluster/ClusterContext.cs | 18 ++-- .../Cluster/ClusterOperations.cs | 8 +- .../BDN.benchmark/Custom/CustomProcSet.cs | 50 ++++++++++ .../CustomTxnSet.cs} | 8 +- .../Operations/CustomOperations.cs | 93 +++++++++++++++++++ .../Operations/ObjectOperations.cs | 37 -------- 7 files changed, 165 insertions(+), 58 deletions(-) create mode 100644 benchmark/BDN.benchmark/Custom/CustomProcSet.cs rename benchmark/BDN.benchmark/{CustomProcs/CustomProcSet.cs => Custom/CustomTxnSet.cs} (90%) create mode 100644 benchmark/BDN.benchmark/Operations/CustomOperations.cs diff --git a/benchmark/BDN.benchmark/BDN.benchmark.csproj b/benchmark/BDN.benchmark/BDN.benchmark.csproj index 2965fbcf83..21d1589cc3 100644 --- a/benchmark/BDN.benchmark/BDN.benchmark.csproj +++ b/benchmark/BDN.benchmark/BDN.benchmark.csproj @@ -22,9 +22,10 @@ - - - - + + + + + diff --git a/benchmark/BDN.benchmark/Cluster/ClusterContext.cs b/benchmark/BDN.benchmark/Cluster/ClusterContext.cs index 4215e0da68..1ae7fc4540 100644 --- a/benchmark/BDN.benchmark/Cluster/ClusterContext.cs +++ b/benchmark/BDN.benchmark/Cluster/ClusterContext.cs @@ -20,7 +20,7 @@ unsafe class ClusterContext public static ReadOnlySpan keyTag => "{0}"u8; public Request[] singleGetSet; public Request[] singleMGetMSet; - public Request singleCPBSET; + public Request singleCTXNSET; public void Dispose() { @@ -41,7 +41,7 @@ public void SetupSingleInstance(bool disableSlotVerification = false) opt.CheckpointDir = "/tmp"; server = new EmbeddedRespServer(opt); session = server.GetRespSession(); - _ = server.Register.NewTransactionProc(CustomProcSet.CommandName, () => new CustomProcSet(), new RespCommandsInfo { Arity = CustomProcSet.Arity }); + _ = server.Register.NewTransactionProc(CustomTxnSet.CommandName, () => new CustomTxnSet(), new RespCommandsInfo { Arity = CustomTxnSet.Arity }); } public void AddSlotRange(List<(int, int)> slotRanges) @@ -134,7 +134,7 @@ public void CreateMGetMSet(int keySize = 8, int valueSize = 32, int batchSize = singleMGetMSet = [mGetReq, mSetReq]; } - public void CreateCPBSET(int keySize = 8, int batchSize = 100) + public void CreateCTXNSET(int keySize = 8, int batchSize = 100) { var keys = new byte[8][]; for (var i = 0; i < 8; i++) @@ -145,22 +145,22 @@ public void CreateCPBSET(int keySize = 8, int batchSize = 100) benchUtils.RandomBytes(ref keys[i], startOffset: keyTag.Length); } - var cpbsetByteCount = "*9\r\n$6\r\nCPBSET\r\n"u8.Length + (8 * (1 + NumUtils.NumDigits(keySize) + 2 + keySize + 2)); - var cpbsetReq = new Request(batchSize * cpbsetByteCount); - var curr = cpbsetReq.ptr; - var end = curr + cpbsetReq.buffer.Length; + var ctxnsetByteCount = "*9\r\n$7\r\nCTXNSET\r\n"u8.Length + (8 * (1 + NumUtils.NumDigits(keySize) + 2 + keySize + 2)); + var ctxnsetReq = new Request(batchSize * ctxnsetByteCount); + var curr = ctxnsetReq.ptr; + var end = curr + ctxnsetReq.buffer.Length; for (var i = 0; i < batchSize; i++) { _ = RespWriteUtils.WriteArrayLength(9, ref curr, end); - _ = RespWriteUtils.WriteBulkString("CPBSET"u8, ref curr, end); + _ = RespWriteUtils.WriteBulkString("CTXNSET"u8, ref curr, end); for (var j = 0; j < 8; j++) { _ = RespWriteUtils.WriteBulkString(keys[j], ref curr, end); } } - singleCPBSET = cpbsetReq; + singleCTXNSET = ctxnsetReq; } public void Consume(byte* ptr, int length) diff --git a/benchmark/BDN.benchmark/Cluster/ClusterOperations.cs b/benchmark/BDN.benchmark/Cluster/ClusterOperations.cs index 3602060a12..2799885ae9 100644 --- a/benchmark/BDN.benchmark/Cluster/ClusterOperations.cs +++ b/benchmark/BDN.benchmark/Cluster/ClusterOperations.cs @@ -36,14 +36,14 @@ public virtual void GlobalSetup() cc.AddSlotRange([(0, 16383)]); cc.CreateGetSet(); cc.CreateMGetMSet(); - cc.CreateCPBSET(); + cc.CreateCTXNSET(); // Warmup/Prepopulate stage cc.Consume(cc.singleGetSet[1].ptr, cc.singleGetSet[1].buffer.Length); // Warmup/Prepopulate stage cc.Consume(cc.singleMGetMSet[1].ptr, cc.singleMGetMSet[1].buffer.Length); // Warmup/Prepopulate stage - cc.Consume(cc.singleCPBSET.ptr, cc.singleCPBSET.buffer.Length); + cc.Consume(cc.singleCTXNSET.ptr, cc.singleCTXNSET.buffer.Length); } [GlobalCleanup] @@ -77,9 +77,9 @@ public void MSet() } [Benchmark] - public void CPBSET() + public void CTXNSET() { - cc.Consume(cc.singleCPBSET.ptr, cc.singleCPBSET.buffer.Length); + cc.Consume(cc.singleCTXNSET.ptr, cc.singleCTXNSET.buffer.Length); } } } \ No newline at end of file diff --git a/benchmark/BDN.benchmark/Custom/CustomProcSet.cs b/benchmark/BDN.benchmark/Custom/CustomProcSet.cs new file mode 100644 index 0000000000..7e2b482a96 --- /dev/null +++ b/benchmark/BDN.benchmark/Custom/CustomProcSet.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using Garnet.common; +using Garnet.server; + +namespace BDN.benchmark.CustomProcs +{ + class CustomProcSet : CustomProcedure + { + /// + /// Parameters including command + /// + public const int Arity = 9; + + /// + /// Command name + /// + public const string CommandName = "CPROCSET"; + + /// + /// CPROCSET key1 key2 key3 key4 value1 value2 value3 value4 + /// + /// + /// + /// + /// + /// + public override bool Execute(TGarnetApi api, ref CustomProcedureInput procInput, ref MemoryResult output) + { + var offset = 0; + var setA = GetNextArg(ref procInput, ref offset); + var setB = GetNextArg(ref procInput, ref offset); + var setC = GetNextArg(ref procInput, ref offset); + var setD = GetNextArg(ref procInput, ref offset); + + var valueA = GetNextArg(ref procInput, ref offset); + var valueB = GetNextArg(ref procInput, ref offset); + var valueC = GetNextArg(ref procInput, ref offset); + var valueD = GetNextArg(ref procInput, ref offset); + + _ = api.SET(setA, valueA); + _ = api.SET(setB, valueB); + _ = api.SET(setC, valueC); + _ = api.SET(setD, valueD); + + return true; + } + } +} \ No newline at end of file diff --git a/benchmark/BDN.benchmark/CustomProcs/CustomProcSet.cs b/benchmark/BDN.benchmark/Custom/CustomTxnSet.cs similarity index 90% rename from benchmark/BDN.benchmark/CustomProcs/CustomProcSet.cs rename to benchmark/BDN.benchmark/Custom/CustomTxnSet.cs index d602f0d6c4..b833e696c5 100644 --- a/benchmark/BDN.benchmark/CustomProcs/CustomProcSet.cs +++ b/benchmark/BDN.benchmark/Custom/CustomTxnSet.cs @@ -8,9 +8,9 @@ namespace BDN.benchmark.CustomProcs { /// - /// Custom procedure to set values + /// Custom transaction to set values /// - sealed class CustomProcSet : CustomTransactionProcedure + sealed class CustomTxnSet : CustomTransactionProcedure { /// /// Parameters including command @@ -20,7 +20,7 @@ sealed class CustomProcSet : CustomTransactionProcedure /// /// Command name /// - public const string CommandName = "CPBSET"; + public const string CommandName = "CTXNSET"; ArgSlice setA; ArgSlice setB; @@ -33,7 +33,7 @@ sealed class CustomProcSet : CustomTransactionProcedure ArgSlice valueD; /// - /// CPBSET key1 key2 key3 key4 value1 value2 value3 value4 + /// CTXNSET key1 key2 key3 key4 value1 value2 value3 value4 /// /// /// diff --git a/benchmark/BDN.benchmark/Operations/CustomOperations.cs b/benchmark/BDN.benchmark/Operations/CustomOperations.cs new file mode 100644 index 0000000000..eea46648c2 --- /dev/null +++ b/benchmark/BDN.benchmark/Operations/CustomOperations.cs @@ -0,0 +1,93 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +using BDN.benchmark.CustomProcs; +using BenchmarkDotNet.Attributes; +using Garnet; +using Garnet.server; + +namespace BDN.benchmark.Operations +{ + /// + /// Benchmark for ObjectOperations + /// + [MemoryDiagnoser] + public unsafe class CustomOperations : OperationsBase + { + static ReadOnlySpan SETIFPM => "*4\r\n$7\r\nSETIFPM\r\n$1\r\nk\r\n$3\r\nval\r\n$1\r\nv\r\n"u8; + byte[] setIfPmRequestBuffer; + byte* setIfPmRequestBufferPointer; + + static ReadOnlySpan MYDICTSETGET => "*4\r\n$9\r\nMYDICTSET\r\n$2\r\nck\r\n$1\r\nf\r\n$1\r\nv\r\n*3\r\n$9\r\nMYDICTGET\r\n$2\r\nck\r\n$1\r\nf\r\n"u8; + byte[] myDictSetGetRequestBuffer; + byte* myDictSetGetRequestBufferPointer; + + static ReadOnlySpan CTXNSET => "*9\r\n$7\r\nCTXNSET\r\n$6\r\n{0}000\r\n$6\r\n{0}001\r\n$6\r\n{0}002\r\n$6\r\n{0}003\r\n$6\r\n{0}000\r\n$6\r\n{0}001\r\n$6\r\n{0}002\r\n$6\r\n{0}003\r\n"u8; + byte[] ctxnsetBuffer; + byte* ctxnsetBufferPointer; + + static ReadOnlySpan CPROCSET => "*9\r\n$8\r\nCPROCSET\r\n$6\r\n{0}000\r\n$6\r\n{0}001\r\n$6\r\n{0}002\r\n$6\r\n{0}003\r\n$6\r\n{0}000\r\n$6\r\n{0}001\r\n$6\r\n{0}002\r\n$6\r\n{0}003\r\n"u8; + byte[] cprocsetBuffer; + byte* cprocsetBufferPointer; + + void CreateExtensions() + { + // Register custom raw string command + server.Register.NewCommand("SETIFPM", CommandType.ReadModifyWrite, new SetIfPMCustomCommand(), new RespCommandsInfo { Arity = 4 }); + + // Register custom object type and commands + var factory = new MyDictFactory(); + server.Register.NewType(factory); + server.Register.NewCommand("MYDICTSET", CommandType.ReadModifyWrite, factory, new MyDictSet(), new RespCommandsInfo { Arity = 4 }); + server.Register.NewCommand("MYDICTGET", CommandType.Read, factory, new MyDictGet(), new RespCommandsInfo { Arity = 3 }); + + // Register custom transaction + server.Register.NewTransactionProc(CustomProcs.CustomTxnSet.CommandName, () => new CustomTxnSet(), + new RespCommandsInfo { Arity = CustomProcs.CustomTxnSet.Arity }); + + // Register custom procedure + server.Register.NewProcedure(CustomProcs.CustomProcSet.CommandName, new CustomProcSet(), + new RespCommandsInfo { Arity = CustomProcs.CustomProcSet.Arity }); + } + + public override void GlobalSetup() + { + base.GlobalSetup(); + CreateExtensions(); + + SetupOperation(ref setIfPmRequestBuffer, ref setIfPmRequestBufferPointer, SETIFPM); + SetupOperation(ref myDictSetGetRequestBuffer, ref myDictSetGetRequestBufferPointer, MYDICTSETGET); + SetupOperation(ref ctxnsetBuffer, ref ctxnsetBufferPointer, CTXNSET); + SetupOperation(ref cprocsetBuffer, ref cprocsetBufferPointer, CPROCSET); + + SlowConsumeMessage("*4\r\n$7\r\nSETIFPM\r\n$1\r\nk\r\n$3\r\nval\r\n$1\r\nv\r\n"u8); + SlowConsumeMessage("*4\r\n$9\r\nMYDICTSET\r\n$2\r\nck\r\n$1\r\nf\r\n$1\r\nv\r\n"u8); + SlowConsumeMessage(ctxnsetBuffer); + SlowConsumeMessage(cprocsetBuffer); + } + + [Benchmark] + public void CustomRawStringCommand() + { + _ = session.TryConsumeMessages(setIfPmRequestBufferPointer, setIfPmRequestBuffer.Length); + } + + [Benchmark] + public void CustomObjectCommand() + { + _ = session.TryConsumeMessages(myDictSetGetRequestBufferPointer, myDictSetGetRequestBuffer.Length); + } + + [Benchmark] + public void CustomTransaction() + { + _ = session.TryConsumeMessages(ctxnsetBufferPointer, ctxnsetBuffer.Length); + } + + [Benchmark] + public void CustomProcedure() + { + _ = session.TryConsumeMessages(cprocsetBufferPointer, cprocsetBuffer.Length); + } + } +} \ No newline at end of file diff --git a/benchmark/BDN.benchmark/Operations/ObjectOperations.cs b/benchmark/BDN.benchmark/Operations/ObjectOperations.cs index 8dcd7ffa21..19b9fd52dc 100644 --- a/benchmark/BDN.benchmark/Operations/ObjectOperations.cs +++ b/benchmark/BDN.benchmark/Operations/ObjectOperations.cs @@ -1,10 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. -using BDN.benchmark.CustomProcs; using BenchmarkDotNet.Attributes; -using Garnet; -using Garnet.server; namespace BDN.benchmark.Operations { @@ -30,41 +27,19 @@ public unsafe class ObjectOperations : OperationsBase byte[] hSetDelRequestBuffer; byte* hSetDelRequestBufferPointer; - static ReadOnlySpan MYDICTSETGET => "*4\r\n$9\r\nMYDICTSET\r\n$2\r\nck\r\n$1\r\nf\r\n$1\r\nv\r\n*3\r\n$9\r\nMYDICTGET\r\n$2\r\nck\r\n$1\r\nf\r\n"u8; - byte[] myDictSetGetRequestBuffer; - byte* myDictSetGetRequestBufferPointer; - - static ReadOnlySpan CPBSET => "*9\r\n$6\r\nCPBSET\r\n$6\r\n{0}000\r\n$6\r\n{0}001\r\n$6\r\n{0}002\r\n$6\r\n{0}003\r\n$6\r\n{0}000\r\n$6\r\n{0}001\r\n$6\r\n{0}002\r\n$6\r\n{0}003\r\n"u8; - byte[] cpbsetBuffer; - byte* cpbsetBufferPointer; - - void CreateExtensions() - { - var factory = new MyDictFactory(); - server.Register.NewType(factory); - server.Register.NewCommand("MYDICTSET", CommandType.ReadModifyWrite, factory, new MyDictSet(), new RespCommandsInfo { Arity = 4 }); - server.Register.NewCommand("MYDICTGET", CommandType.Read, factory, new MyDictGet(), new RespCommandsInfo { Arity = 3 }); - server.Register.NewTransactionProc(CustomProcs.CustomProcSet.CommandName, () => new CustomProcSet(), new RespCommandsInfo { Arity = CustomProcs.CustomProcSet.Arity }); - } - public override void GlobalSetup() { base.GlobalSetup(); - CreateExtensions(); SetupOperation(ref zAddRemRequestBuffer, ref zAddRemRequestBufferPointer, ZADDREM); SetupOperation(ref lPushPopRequestBuffer, ref lPushPopRequestBufferPointer, LPUSHPOP); SetupOperation(ref sAddRemRequestBuffer, ref sAddRemRequestBufferPointer, SADDREM); SetupOperation(ref hSetDelRequestBuffer, ref hSetDelRequestBufferPointer, HSETDEL); - SetupOperation(ref myDictSetGetRequestBuffer, ref myDictSetGetRequestBufferPointer, MYDICTSETGET); - SetupOperation(ref cpbsetBuffer, ref cpbsetBufferPointer, CPBSET); // Pre-populate data SlowConsumeMessage("*4\r\n$4\r\nZADD\r\n$1\r\nc\r\n$1\r\n1\r\n$1\r\nd\r\n"u8); SlowConsumeMessage("*3\r\n$5\r\nLPUSH\r\n$1\r\nd\r\n$1\r\nf\r\n"u8); SlowConsumeMessage("*3\r\n$4\r\nSADD\r\n$1\r\ne\r\n$1\r\nb\r\n"u8); SlowConsumeMessage("*3\r\n$4\r\nHSET\r\n$1\r\nf\r\n$1\r\nb\r\n$1\r\nb\r\n"u8); - SlowConsumeMessage("*4\r\n$9\r\nMYDICTSET\r\n$2\r\nck\r\n$1\r\nf\r\n$1\r\nv\r\n"u8); - SlowConsumeMessage(cpbsetBuffer); } [Benchmark] @@ -90,17 +65,5 @@ public void HSetDel() { _ = session.TryConsumeMessages(hSetDelRequestBufferPointer, hSetDelRequestBuffer.Length); } - - [Benchmark] - public void MyDictSetGet() - { - _ = session.TryConsumeMessages(myDictSetGetRequestBufferPointer, myDictSetGetRequestBuffer.Length); - } - - [Benchmark] - public void CustomProcSet() - { - _ = session.TryConsumeMessages(cpbsetBufferPointer, cpbsetBuffer.Length); - } } } \ No newline at end of file From b2448121ce2ddc3c0850cfd12dea25c42fb3e529 Mon Sep 17 00:00:00 2001 From: Tal Zaccai Date: Fri, 25 Oct 2024 15:23:26 -0700 Subject: [PATCH 12/15] Enabling limited parameter access to SessionParseState (#744) * wip * format * Small fixes * Fixing slice to return a new SPS struct * format * Fixing comments * fix * removing test * Adding WriteDirectLargeRespString * fix * format * fixing comments * Removing unnecessary constructors --- libs/common/RespWriteUtils.cs | 29 ++++ libs/server/API/GarnetWatchApi.cs | 6 +- libs/server/Custom/CustomCommandUtils.cs | 32 ++-- libs/server/Custom/CustomFunctions.cs | 21 ++- libs/server/Custom/CustomRespCommands.cs | 12 +- libs/server/InputHeader.cs | 108 ++++++-------- libs/server/Objects/Hash/HashObjectImpl.cs | 34 ++--- libs/server/Objects/List/ListObjectImpl.cs | 26 ++-- libs/server/Objects/ObjectUtils.cs | 4 +- libs/server/Objects/Set/SetObjectImpl.cs | 24 ++- .../Objects/SortedSet/SortedSetObjectImpl.cs | 50 +++---- .../SortedSetGeo/SortedSetGeoObjectImpl.cs | 41 ++--- libs/server/Resp/ArrayCommands.cs | 8 +- libs/server/Resp/BasicCommands.cs | 15 +- libs/server/Resp/Bitmap/BitmapCommands.cs | 8 +- libs/server/Resp/HyperLogLog/HyperLogLog.cs | 13 +- .../Resp/HyperLogLog/HyperLogLogCommands.cs | 6 +- libs/server/Resp/KeyAdminCommands.cs | 4 +- libs/server/Resp/Objects/HashCommands.cs | 16 +- libs/server/Resp/Objects/ListCommands.cs | 10 +- libs/server/Resp/Objects/SetCommands.cs | 6 +- .../Resp/Objects/SharedObjectCommands.cs | 4 +- libs/server/Resp/Objects/SortedSetCommands.cs | 20 +-- .../Resp/Objects/SortedSetGeoCommands.cs | 5 +- libs/server/Resp/Parser/SessionParseState.cs | 140 ++++++++++-------- libs/server/Resp/RespServerSession.cs | 12 ++ .../Functions/MainStore/PrivateMethods.cs | 34 ++--- .../Storage/Functions/MainStore/RMWMethods.cs | 75 +++++----- .../Functions/MainStore/VarLenInputMethods.cs | 30 ++-- .../Functions/ObjectStore/RMWMethods.cs | 4 +- .../Session/MainStore/HyperLogLogOps.cs | 38 +++-- .../Storage/Session/MainStore/MainStoreOps.cs | 8 +- .../Storage/Session/ObjectStore/Common.cs | 2 +- .../Storage/Session/ObjectStore/SetOps.cs | 3 +- .../Session/ObjectStore/SortedSetOps.cs | 2 +- test/Garnet.test/DeleteTxn.cs | 4 +- test/Garnet.test/ObjectExpiryTxn.cs | 6 +- test/Garnet.test/RespAdminCommandsTests.cs | 10 ++ test/Garnet.test/SortedSetRemoveTxn.cs | 6 +- test/Garnet.test/TestProcedureBitmap.cs | 20 +-- test/Garnet.test/TestProcedureHLL.cs | 6 +- test/Garnet.test/TestProcedureHash.cs | 10 +- 42 files changed, 459 insertions(+), 453 deletions(-) diff --git a/libs/common/RespWriteUtils.cs b/libs/common/RespWriteUtils.cs index c2bb77585d..557fc09fba 100644 --- a/libs/common/RespWriteUtils.cs +++ b/libs/common/RespWriteUtils.cs @@ -289,6 +289,22 @@ public static bool WriteDirect(ref T item, ref byte* curr, byte* end) where T return true; } + /// + /// Write length header of bulk string + /// + public static bool WriteBulkStringLength(ReadOnlySpan item, ref byte* curr, byte* end) + { + var itemDigits = NumUtils.NumDigits(item.Length); + var totalLen = 1 + itemDigits + 2; + if (totalLen > (int)(end - curr)) + return false; + + *curr++ = (byte)'$'; + NumUtils.IntToBytes(item.Length, itemDigits, ref curr); + WriteNewline(ref curr); + return true; + } + /// /// Write bulk string /// @@ -349,6 +365,19 @@ public static bool WriteUtf8BulkString(ReadOnlySpan chars, ref byte* curr, return true; } + /// + /// Write new line + /// + public static bool WriteNewLine(ref byte* curr, byte* end) + { + var totalLen = 2; + if (totalLen > (int)(end - curr)) + return false; + + WriteNewline(ref curr); + return true; + } + /// /// Get length of bulk string /// diff --git a/libs/server/API/GarnetWatchApi.cs b/libs/server/API/GarnetWatchApi.cs index ef6d82266f..40cc09678c 100644 --- a/libs/server/API/GarnetWatchApi.cs +++ b/libs/server/API/GarnetWatchApi.cs @@ -486,12 +486,12 @@ public GarnetStatus StringBitFieldReadOnly(ref SpanByte key, ref RawStringInput /// public GarnetStatus HyperLogLogLength(ref RawStringInput input, out long count, out bool error) { - var currTokenIdx = input.parseStateFirstArgIdx; - while (currTokenIdx < input.parseState.Count) + for (var i = 0; i < input.parseState.Count; i++) { - var key = input.parseState.GetArgSliceByRef(currTokenIdx++); + var key = input.parseState.GetArgSliceByRef(i); garnetApi.WATCH(key, StoreType.Main); } + return garnetApi.HyperLogLogLength(ref input, out count, out error); } diff --git a/libs/server/Custom/CustomCommandUtils.cs b/libs/server/Custom/CustomCommandUtils.cs index 9770e68ef1..d1acc7b4f7 100644 --- a/libs/server/Custom/CustomCommandUtils.cs +++ b/libs/server/Custom/CustomCommandUtils.cs @@ -24,8 +24,8 @@ public static class CustomCommandUtils /// public static ReadOnlySpan GetFirstArg(ref ObjectInput input) { - var offset = 0; - return GetNextArg(ref input, ref offset); + var idx = 0; + return GetNextArg(ref input, ref idx); } /// @@ -35,37 +35,37 @@ public static ReadOnlySpan GetFirstArg(ref ObjectInput input) /// public static ReadOnlySpan GetFirstArg(ref RawStringInput input) { - var offset = 0; - return GetNextArg(ref input, ref offset); + var idx = 0; + return GetNextArg(ref input, ref idx); } /// - /// Get argument from input, at specified offset (starting from 0) + /// Get argument from input, at specified index (starting from 0) /// /// Object store input - /// Current offset into input + /// Current argument index in input /// Argument as a span - public static ReadOnlySpan GetNextArg(ref ObjectInput input, scoped ref int offset) + public static ReadOnlySpan GetNextArg(ref ObjectInput input, scoped ref int idx) { - var arg = input.parseStateFirstArgIdx + offset < input.parseState.Count - ? input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx + offset).ReadOnlySpan + var arg = idx < input.parseState.Count + ? input.parseState.GetArgSliceByRef(idx).ReadOnlySpan : default; - offset++; + idx++; return arg; } /// - /// Get argument from input, at specified offset (starting from 0) + /// Get argument from input, at specified index (starting from 0) /// /// Main store input - /// Current offset into input + /// Current argument index in input /// Argument as a span - public static ReadOnlySpan GetNextArg(ref RawStringInput input, scoped ref int offset) + public static ReadOnlySpan GetNextArg(ref RawStringInput input, scoped ref int idx) { - var arg = input.parseStateFirstArgIdx + offset < input.parseState.Count - ? input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx + offset).ReadOnlySpan + var arg = idx < input.parseState.Count + ? input.parseState.GetArgSliceByRef(idx).ReadOnlySpan : default; - offset++; + idx++; return arg; } diff --git a/libs/server/Custom/CustomFunctions.cs b/libs/server/Custom/CustomFunctions.cs index a87592e1b5..70bc330632 100644 --- a/libs/server/Custom/CustomFunctions.cs +++ b/libs/server/Custom/CustomFunctions.cs @@ -183,30 +183,29 @@ protected static unsafe void WriteError(ref MemoryResult output, ReadOnlyS } /// - /// Get argument from input, at specified offset (starting from 0) + /// Get argument from parse state, at specified index (starting from 0) /// /// Current parse state - /// - /// Current offset into parse state + /// Current argument index in parse state /// Argument as a span - protected static unsafe ArgSlice GetNextArg(ref SessionParseState parseState, int parseStateFirstArgIdx, ref int offset) + protected static unsafe ArgSlice GetNextArg(ref SessionParseState parseState, ref int idx) { - var arg = parseStateFirstArgIdx + offset < parseState.Count - ? parseState.GetArgSliceByRef(parseStateFirstArgIdx + offset) + var arg = idx < parseState.Count + ? parseState.GetArgSliceByRef(idx) : default; - offset++; + idx++; return arg; } /// - /// Get argument from input, at specified offset (starting from 0) + /// Get argument from input, at specified index (starting from 0) /// /// Procedure input - /// Current offset into parse state + /// Current argument index in parse state /// Argument as a span - protected static unsafe ArgSlice GetNextArg(ref CustomProcedureInput procInput, ref int offset) + protected static unsafe ArgSlice GetNextArg(ref CustomProcedureInput procInput, ref int idx) { - return GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + return GetNextArg(ref procInput.parseState, ref idx); } } } \ No newline at end of file diff --git a/libs/server/Custom/CustomRespCommands.cs b/libs/server/Custom/CustomRespCommands.cs index d625f66733..26820f8a7d 100644 --- a/libs/server/Custom/CustomRespCommands.cs +++ b/libs/server/Custom/CustomRespCommands.cs @@ -14,7 +14,7 @@ namespace Garnet.server /// internal sealed unsafe partial class RespServerSession : ServerSessionBase { - private bool TryTransactionProc(byte id, CustomTransactionProcedure proc, int parseStateFirstArgIdx = 0, int parseStateLastArgIdx = -1) + private bool TryTransactionProc(byte id, CustomTransactionProcedure proc, int startIdx = 0) { // Define output var output = new MemoryResult(null, 0); @@ -24,7 +24,7 @@ private bool TryTransactionProc(byte id, CustomTransactionProcedure proc, int pa latencyMetrics?.Start(LatencyMetricsType.TX_PROC_LAT); - var procInput = new CustomProcedureInput(ref parseState, parseStateFirstArgIdx, parseStateLastArgIdx); + var procInput = new CustomProcedureInput(ref parseState, startIdx: startIdx); if (txnManager.RunTransactionProc(id, ref procInput, proc, ref output)) { // Write output to wire @@ -56,13 +56,13 @@ public bool RunTransactionProc(byte id, ref CustomProcedureInput procInput, ref } - private void TryCustomProcedure(CustomProcedure proc, int parseStateFirstArgIdx = 0, int parseStateLastArgIdx = -1) + private void TryCustomProcedure(CustomProcedure proc, int startIdx = 0) { Debug.Assert(proc != null); var output = new MemoryResult(null, 0); - var procInput = new CustomProcedureInput(ref parseState, parseStateFirstArgIdx, parseStateLastArgIdx); + var procInput = new CustomProcedureInput(ref parseState, startIdx: startIdx); if (proc.Execute(basicGarnetApi, ref procInput, ref output)) { if (output.MemoryOwner != null) @@ -90,7 +90,7 @@ private bool TryCustomRawStringCommand(RespCommand cmd, long expirat var sbKey = parseState.GetArgSliceByRef(0).SpanByte; var inputArg = expirationTicks > 0 ? DateTimeOffset.UtcNow.Ticks + expirationTicks : expirationTicks; - var input = new RawStringInput(cmd, ref parseState, 1, -1, inputArg); + var input = new RawStringInput(cmd, ref parseState, startIdx: 1, arg1: inputArg); var output = new SpanByteAndMemory(null); GarnetStatus status; @@ -140,7 +140,7 @@ private bool TryCustomObjectCommand(GarnetObjectType objType, byte s // Prepare input var header = new RespInputHeader(objType) { SubId = subid }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); var output = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(null) }; diff --git a/libs/server/InputHeader.cs b/libs/server/InputHeader.cs index 6625b37a26..3a9ed6c7ee 100644 --- a/libs/server/InputHeader.cs +++ b/libs/server/InputHeader.cs @@ -190,16 +190,6 @@ public struct ObjectInput : IStoreInput /// public int arg2; - /// - /// First index to start reading the parse state parameters array for command execution - /// - public int parseStateFirstArgIdx; - - /// - /// Last index to read in the parse state parameters array for command execution - /// - public int parseStateLastArgIdx; - /// /// Session parse state /// @@ -223,23 +213,32 @@ public ObjectInput(RespInputHeader header, int arg1 = 0, int arg2 = 0) /// /// Input header /// Parse state - /// Index at which to start reading parse state parameters array - /// Last index to read in the parse state parameters array for command execution /// First general-purpose argument /// Second general-purpose argument - public ObjectInput(RespInputHeader header, ref SessionParseState parseState, int parseStateFirstArgIdx = 0, - int parseStateLastArgIdx = -1, int arg1 = 0, int arg2 = 0) + public ObjectInput(RespInputHeader header, ref SessionParseState parseState, int arg1 = 0, int arg2 = 0) : this(header, arg1, arg2) { this.parseState = parseState; - this.parseStateFirstArgIdx = parseStateFirstArgIdx; - this.parseStateLastArgIdx = parseStateLastArgIdx; + } + + /// + /// Create a new instance of ObjectInput + /// + /// Input header + /// Parse state + /// First command argument index in parse state + /// First general-purpose argument + /// Second general-purpose argument + public ObjectInput(RespInputHeader header, ref SessionParseState parseState, int startIdx, int arg1 = 0, int arg2 = 0) + : this(header, arg1, arg2) + { + this.parseState = parseState.Slice(startIdx); } /// public int SerializedLength => header.SpanByte.TotalSize + (2 * sizeof(int)) // arg1 + arg2 - + parseState.GetSerializedLength(parseStateFirstArgIdx, parseStateLastArgIdx); + + parseState.GetSerializedLength(); /// public unsafe int CopyTo(byte* dest, int length) @@ -261,9 +260,8 @@ public unsafe int CopyTo(byte* dest, int length) curr += sizeof(int); // Serialize parse state - // Only serialize arguments starting from parseStateFirstArgIdx var remainingLength = length - (int)(curr - dest); - var len = parseState.CopyTo(curr, parseStateFirstArgIdx, parseStateLastArgIdx, remainingLength); + var len = parseState.CopyTo(curr, remainingLength); curr += len; // Number of serialized bytes @@ -293,8 +291,6 @@ public unsafe int DeserializeFrom(byte* src) var len = parseState.DeserializeFrom(curr); curr += len; - parseStateLastArgIdx = -1; - return (int)(src - curr); } } @@ -314,16 +310,6 @@ public struct RawStringInput : IStoreInput /// public long arg1; - /// - /// Index at which to start reading the parse state parameters array for command execution - /// - public int parseStateFirstArgIdx = 0; - - /// - /// Last index to read in the parse state parameters array for command execution - /// - public int parseStateLastArgIdx = -1; - /// /// Session parse state /// @@ -358,22 +344,30 @@ public RawStringInput(ushort cmd, byte flags = 0, long arg1 = 0) : /// /// Command /// Parse state - /// Index at which to start reading parse state parameters array - /// Last index to read in the parse state parameters array for command execution /// General-purpose argument /// Flags - public RawStringInput(RespCommand cmd, ref SessionParseState parseState, int parseStateFirstArgIdx = 0, - int parseStateLastArgIdx = -1, long arg1 = 0, RespInputFlags flags = 0) : this(cmd, flags, arg1) + public RawStringInput(RespCommand cmd, ref SessionParseState parseState, long arg1 = 0, RespInputFlags flags = 0) : this(cmd, flags, arg1) { this.parseState = parseState; - this.parseStateFirstArgIdx = parseStateFirstArgIdx; - this.parseStateLastArgIdx = parseStateLastArgIdx; + } + + /// + /// Create a new instance of RawStringInput + /// + /// Command + /// Parse state + /// First command argument index in parse state + /// General-purpose argument + /// Flags + public RawStringInput(RespCommand cmd, ref SessionParseState parseState, int startIdx, long arg1 = 0, RespInputFlags flags = 0) : this(cmd, flags, arg1) + { + this.parseState = parseState.Slice(startIdx); } /// public int SerializedLength => header.SpanByte.TotalSize + sizeof(long) // arg1 - + parseState.GetSerializedLength(parseStateFirstArgIdx, parseStateLastArgIdx); + + parseState.GetSerializedLength(); /// public unsafe int CopyTo(byte* dest, int length) @@ -391,9 +385,8 @@ public unsafe int CopyTo(byte* dest, int length) curr += sizeof(long); // Serialize parse state - // Only serialize arguments starting from parseStateFirstArgIdx var remainingLength = length - (int)(curr - dest); - var len = parseState.CopyTo(curr, parseStateFirstArgIdx, parseStateLastArgIdx, remainingLength); + var len = parseState.CopyTo(curr, remainingLength); curr += len; // Serialize length @@ -419,8 +412,6 @@ public unsafe int DeserializeFrom(byte* src) var len = parseState.DeserializeFrom(curr); curr += len; - parseStateLastArgIdx = -1; - return (int)(curr - src); } } @@ -430,16 +421,6 @@ public unsafe int DeserializeFrom(byte* src) /// public struct CustomProcedureInput : IStoreInput { - /// - /// Index at which to start reading the parse state parameters array for command execution - /// - public int parseStateFirstArgIdx; - - /// - /// Last index to read in the parse state parameters array for command execution - /// - public int parseStateLastArgIdx; - /// /// Session parse state /// @@ -449,17 +430,23 @@ public struct CustomProcedureInput : IStoreInput /// Create a new instance of RawStringInput /// /// Parse state - /// Index at which to start reading parse state parameters array - /// Last index to read in the parse state parameters array - public CustomProcedureInput(ref SessionParseState parseState, int parseStateFirstArgIdx = 0, int parseStateLastArgIdx = -1) + public CustomProcedureInput(ref SessionParseState parseState) { this.parseState = parseState; - this.parseStateFirstArgIdx = parseStateFirstArgIdx; - this.parseStateLastArgIdx = parseStateLastArgIdx; + } + + /// + /// Create a new instance of RawStringInput + /// + /// Parse state + /// First command argument index in parse state + public CustomProcedureInput(ref SessionParseState parseState, int startIdx) + { + this.parseState = parseState.Slice(startIdx); } /// - public int SerializedLength => parseState.GetSerializedLength(parseStateFirstArgIdx, parseStateLastArgIdx); + public int SerializedLength => parseState.GetSerializedLength(); /// public unsafe int CopyTo(byte* dest, int length) @@ -469,9 +456,8 @@ public unsafe int CopyTo(byte* dest, int length) var curr = dest; // Serialize parse state - // Only serialize arguments starting from parseStateFirstArgIdx var remainingLength = (int)(curr - dest); - var len = parseState.CopyTo(curr, parseStateFirstArgIdx, parseStateLastArgIdx, remainingLength); + var len = parseState.CopyTo(curr, remainingLength); curr += len; return (int)(curr - dest); @@ -483,8 +469,6 @@ public unsafe int DeserializeFrom(byte* src) // Deserialize parse state var len = parseState.DeserializeFrom(src); - parseStateLastArgIdx = -1; - return len; } } diff --git a/libs/server/Objects/Hash/HashObjectImpl.cs b/libs/server/Objects/Hash/HashObjectImpl.cs index b47040c987..937df4c85e 100644 --- a/libs/server/Objects/Hash/HashObjectImpl.cs +++ b/libs/server/Objects/Hash/HashObjectImpl.cs @@ -28,12 +28,10 @@ private void HashGet(ref ObjectInput input, ref SpanByteAndMemory output) var curr = ptr; var end = curr + output.Length; - var currTokenIdx = input.parseStateFirstArgIdx; - ObjectOutputHeader _output = default; try { - var key = input.parseState.GetArgSliceByRef(currTokenIdx).SpanByte.ToByteArray(); + var key = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); if (hash.TryGetValue(key, out var hashValue)) { @@ -67,19 +65,15 @@ private void HashMultipleGet(ref ObjectInput input, ref SpanByteAndMemory output var curr = ptr; var end = curr + output.Length; - var count = input.parseState.Count; - var currTokenIdx = input.parseStateFirstArgIdx; - ObjectOutputHeader _output = default; try { - var expectedTokenCount = count - input.parseStateFirstArgIdx; - while (!RespWriteUtils.WriteArrayLength(expectedTokenCount, ref curr, end)) + while (!RespWriteUtils.WriteArrayLength(input.parseState.Count, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - while (currTokenIdx < count) + for (var i = 0; i < input.parseState.Count; i++) { - var key = input.parseState.GetArgSliceByRef(currTokenIdx++).SpanByte.ToByteArray(); + var key = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); if (hash.TryGetValue(key, out var hashValue)) { @@ -153,9 +147,9 @@ private void HashDelete(ref ObjectInput input, byte* output) var _output = (ObjectOutputHeader*)output; *_output = default; - for (var currTokenIdx = input.parseStateFirstArgIdx; currTokenIdx < input.parseState.Count; currTokenIdx++) + for (var i = 0; i < input.parseState.Count; i++) { - var key = input.parseState.GetArgSliceByRef(currTokenIdx).SpanByte.ToByteArray(); + var key = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); if (hash.Remove(key, out var hashValue)) { @@ -175,7 +169,7 @@ private void HashStrLength(ref ObjectInput input, byte* output) var _output = (ObjectOutputHeader*)output; *_output = default; - var key = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).SpanByte.ToByteArray(); + var key = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); _output->result1 = hash.TryGetValue(key, out var hashValue) ? hashValue.Length : 0; } @@ -184,7 +178,7 @@ private void HashExists(ref ObjectInput input, byte* output) var _output = (ObjectOutputHeader*)output; *_output = default; - var field = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).SpanByte.ToByteArray(); + var field = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); _output->result1 = hash.ContainsKey(field) ? 1 : 0; } @@ -263,10 +257,10 @@ private void HashSet(ref ObjectInput input, byte* output) *_output = default; var hop = input.header.HashOp; - for (var currIdx = input.parseStateFirstArgIdx; currIdx < input.parseState.Count; currIdx += 2) + for (var i = 0; i < input.parseState.Count; i += 2) { - var key = input.parseState.GetArgSliceByRef(currIdx).SpanByte.ToByteArray(); - var value = input.parseState.GetArgSliceByRef(currIdx + 1).SpanByte.ToByteArray(); + var key = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); + var value = input.parseState.GetArgSliceByRef(i + 1).SpanByte.ToByteArray(); if (!hash.TryGetValue(key, out var hashValue)) { @@ -341,15 +335,13 @@ private void HashIncrement(ref ObjectInput input, ref SpanByteAndMemory output) ObjectOutputHeader _output = default; - var currTokenIdx = input.parseStateFirstArgIdx; - // This value is used to indicate partial command execution _output.result1 = int.MinValue; try { - var key = input.parseState.GetArgSliceByRef(currTokenIdx++).SpanByte.ToByteArray(); - var incrSlice = input.parseState.GetArgSliceByRef(currTokenIdx); + var key = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); + var incrSlice = input.parseState.GetArgSliceByRef(1); var valueExists = hash.TryGetValue(key, out var value); if (op == HashOperation.HINCRBY) diff --git a/libs/server/Objects/List/ListObjectImpl.cs b/libs/server/Objects/List/ListObjectImpl.cs index 31cfd988eb..ecca643c6a 100644 --- a/libs/server/Objects/List/ListObjectImpl.cs +++ b/libs/server/Objects/List/ListObjectImpl.cs @@ -27,7 +27,7 @@ private void ListRemove(ref ObjectInput input, byte* output) _output->result1 = int.MinValue; // get the source string to remove - var itemSpan = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).ReadOnlySpan; + var itemSpan = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; var removedCount = 0; _output->result1 = 0; @@ -83,16 +83,14 @@ private void ListInsert(ref ObjectInput input, byte* output) if (list.Count > 0) { - var currTokenIdx = input.parseStateFirstArgIdx; - // figure out where to insert BEFORE or AFTER - var position = input.parseState.GetArgSliceByRef(currTokenIdx++).ReadOnlySpan; + var position = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; // get the source string - var pivot = input.parseState.GetArgSliceByRef(currTokenIdx++).ReadOnlySpan; + var pivot = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; // get the string to INSERT into the list - var item = input.parseState.GetArgSliceByRef(currTokenIdx).SpanByte.ToByteArray(); + var item = input.parseState.GetArgSliceByRef(2).SpanByte.ToByteArray(); var insertBefore = position.SequenceEqual(CmdStrings.BEFORE); @@ -282,9 +280,9 @@ private void ListPush(ref ObjectInput input, byte* output, bool fAddAtHead) *_output = default; _output->result1 = 0; - for (var currTokenIdx = input.parseStateFirstArgIdx; currTokenIdx < input.parseState.Count; currTokenIdx++) + for (var i = 0; i < input.parseState.Count; i++) { - var value = input.parseState.GetArgSliceByRef(currTokenIdx).SpanByte.ToByteArray(); + var value = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); // Add the value to the top of the list if (fAddAtHead) @@ -367,7 +365,6 @@ private void ListSet(ref ObjectInput input, ref SpanByteAndMemory output) var output_end = output_currptr + output.Length; ObjectOutputHeader _output = default; - var currTokenIdx = input.parseStateFirstArgIdx; try { if (list.Count == 0) @@ -378,7 +375,7 @@ private void ListSet(ref ObjectInput input, ref SpanByteAndMemory output) } // index - if (!input.parseState.TryGetInt(currTokenIdx++, out var index)) + if (!input.parseState.TryGetInt(0, out var index)) { while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER, ref output_currptr, output_end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref output_startptr, ref ptrHandle, ref output_currptr, ref output_end); @@ -395,7 +392,7 @@ private void ListSet(ref ObjectInput input, ref SpanByteAndMemory output) } // element - var element = input.parseState.GetArgSliceByRef(currTokenIdx).SpanByte.ToByteArray(); + var element = input.parseState.GetArgSliceByRef(1).SpanByte.ToByteArray(); var targetNode = index == 0 ? list.First : (index == list.Count - 1 ? list.Last @@ -422,8 +419,7 @@ private void ListSet(ref ObjectInput input, ref SpanByteAndMemory output) private void ListPosition(ref ObjectInput input, ref SpanByteAndMemory output) { - var element = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).ReadOnlySpan; - input.parseStateFirstArgIdx++; + var element = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; var isMemory = false; MemoryHandle ptrHandle = default; @@ -582,8 +578,6 @@ private void ListPosition(ref ObjectInput input, ref SpanByteAndMemory output) private static unsafe bool ReadListPositionInput(ref ObjectInput input, out int rank, out int count, out bool isDefaultCount, out int maxlen, out ReadOnlySpan error) { - var currTokenIdx = input.parseStateFirstArgIdx; - rank = 1; // By default, LPOS takes first match element count = 1; // By default, LPOS return 1 element isDefaultCount = true; @@ -591,6 +585,8 @@ private static unsafe bool ReadListPositionInput(ref ObjectInput input, out int error = default; + var currTokenIdx = 1; + while (currTokenIdx < input.parseState.Count) { var sbParam = input.parseState.GetArgSliceByRef(currTokenIdx++).ReadOnlySpan; diff --git a/libs/server/Objects/ObjectUtils.cs b/libs/server/Objects/ObjectUtils.cs index dd5f3c63ce..bb9a050667 100644 --- a/libs/server/Objects/ObjectUtils.cs +++ b/libs/server/Objects/ObjectUtils.cs @@ -51,8 +51,6 @@ public static unsafe void ReallocateOutput(ref SpanByteAndMemory output, ref boo public static unsafe bool ReadScanInput(ref ObjectInput input, ref SpanByteAndMemory output, out int cursorInput, out byte* pattern, out int patternLength, out int countInInput, out bool isNoValue, out ReadOnlySpan error) { - var currTokenIdx = input.parseStateFirstArgIdx; - // Cursor cursorInput = input.arg1; @@ -68,6 +66,8 @@ public static unsafe bool ReadScanInput(ref ObjectInput input, ref SpanByteAndMe error = default; isNoValue = false; + var currTokenIdx = 0; + while (currTokenIdx < input.parseState.Count) { var sbParam = input.parseState.GetArgSliceByRef(currTokenIdx++).ReadOnlySpan; diff --git a/libs/server/Objects/Set/SetObjectImpl.cs b/libs/server/Objects/Set/SetObjectImpl.cs index fc20f57218..c6a215a1d2 100644 --- a/libs/server/Objects/Set/SetObjectImpl.cs +++ b/libs/server/Objects/Set/SetObjectImpl.cs @@ -20,9 +20,9 @@ private void SetAdd(ref ObjectInput input, byte* output) var _output = (ObjectOutputHeader*)output; *_output = default; - for (var currTokenIdx = input.parseStateFirstArgIdx; currTokenIdx < input.parseState.Count; currTokenIdx++) + for (var i = 0; i < input.parseState.Count; i++) { - var member = input.parseState.GetArgSliceByRef(currTokenIdx).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); if (set.Add(member)) { @@ -76,7 +76,7 @@ private void SetIsMember(ref ObjectInput input, ref SpanByteAndMemory output) ObjectOutputHeader _output = default; try { - var member = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); var isMember = set.Contains(member); while (!RespWriteUtils.WriteInteger(isMember ? 1 : 0, ref curr, end)) @@ -105,22 +105,19 @@ private void SetMultiIsMember(ref ObjectInput input, ref SpanByteAndMemory outpu ObjectOutputHeader _output = default; try { - var totalCount = input.parseState.Count - input.parseStateFirstArgIdx; - while (!RespWriteUtils.WriteArrayLength(totalCount, ref curr, end)) + while (!RespWriteUtils.WriteArrayLength(input.parseState.Count, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - var argCurr = input.parseStateFirstArgIdx; - while (argCurr < input.parseState.Count) + for (var i = 0; i < input.parseState.Count; i++) { - var member = input.parseState.GetArgSliceByRef(argCurr).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); var isMember = set.Contains(member); while (!RespWriteUtils.WriteInteger(isMember ? 1 : 0, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - - argCurr++; } - _output.result1 = totalCount; + + _output.result1 = input.parseState.Count; } finally { @@ -137,10 +134,9 @@ private void SetRemove(ref ObjectInput input, byte* output) var _output = (ObjectOutputHeader*)output; *_output = default; - var currTokenIdx = input.parseStateFirstArgIdx; - while (currTokenIdx < input.parseState.Count) + for (var i = 0; i < input.parseState.Count; i++) { - var field = input.parseState.GetArgSliceByRef(currTokenIdx++).SpanByte.ToByteArray(); + var field = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); if (set.Remove(field)) { diff --git a/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs b/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs index aef02f9cc8..7c92e3af3c 100644 --- a/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs +++ b/libs/server/Objects/SortedSet/SortedSetObjectImpl.cs @@ -131,7 +131,7 @@ private void SortedSetAdd(ref ObjectInput input, ref SpanByteAndMemory output) try { var options = SortedSetAddOption.None; - var currTokenIdx = input.parseStateFirstArgIdx; + var currTokenIdx = 0; var parsedOptions = false; while (currTokenIdx < input.parseState.Count) @@ -233,9 +233,9 @@ private void SortedSetRemove(ref ObjectInput input, byte* output) var _output = (ObjectOutputHeader*)output; *_output = default; - for (var currIdx = input.parseStateFirstArgIdx; currIdx < input.parseState.Count; currIdx++) + for (var i = 0; i < input.parseState.Count; i++) { - var value = input.parseState.GetArgSliceByRef(currIdx).ReadOnlySpan; + var value = input.parseState.GetArgSliceByRef(i).ReadOnlySpan; var valueArray = value.ToArray(); if (!sortedSetDict.TryGetValue(valueArray, out var key)) @@ -266,8 +266,7 @@ private void SortedSetScore(ref ObjectInput input, ref SpanByteAndMemory output) var curr = ptr; var end = curr + output.Length; - var currIdx = input.parseStateFirstArgIdx; - var member = input.parseState.GetArgSliceByRef(currIdx).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); ObjectOutputHeader outputHeader = default; try @@ -310,12 +309,12 @@ private void SortedSetScores(ref ObjectInput input, ref SpanByteAndMemory output try { - while (!RespWriteUtils.WriteArrayLength(count - 1, ref curr, end)) + while (!RespWriteUtils.WriteArrayLength(count, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - for (var currIdx = input.parseStateFirstArgIdx; currIdx < count; currIdx++) + for (var i = 0; i < count; i++) { - var member = input.parseState.GetArgSliceByRef(currIdx).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); if (!sortedSetDict.TryGetValue(member, out var score)) { @@ -349,11 +348,9 @@ private void SortedSetCount(ref ObjectInput input, ref SpanByteAndMemory output) var curr = ptr; var end = curr + output.Length; - var currIdx = input.parseStateFirstArgIdx; - // Read min & max - var minParamSpan = input.parseState.GetArgSliceByRef(currIdx++).ReadOnlySpan; - var maxParamSpan = input.parseState.GetArgSliceByRef(currIdx).ReadOnlySpan; + var minParamSpan = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + var maxParamSpan = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; ObjectOutputHeader outputHeader = default; @@ -403,14 +400,12 @@ private void SortedSetIncrement(ref ObjectInput input, ref SpanByteAndMemory out var curr = ptr; var end = curr + output.Length; - var currIdx = input.parseStateFirstArgIdx; - ObjectOutputHeader outputHeader = default; try { // Try to read increment value - if (!input.parseState.TryGetDouble(currIdx++, out var incrValue)) + if (!input.parseState.TryGetDouble(0, out var incrValue)) { while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_NOT_VALID_FLOAT, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); @@ -418,7 +413,7 @@ private void SortedSetIncrement(ref ObjectInput input, ref SpanByteAndMemory out } // Read member - var member = input.parseState.GetArgSliceByRef(currIdx).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(1).SpanByte.ToByteArray(); if (sortedSetDict.TryGetValue(member, out var score)) { @@ -463,7 +458,7 @@ private void SortedSetRange(ref ObjectInput input, ref SpanByteAndMemory output) var curr = ptr; var end = curr + output.Length; - var currIdx = input.parseStateFirstArgIdx; + var currIdx = 0; ObjectOutputHeader _output = default; try @@ -680,14 +675,12 @@ private void SortedSetRemoveRangeByRank(ref ObjectInput input, ref SpanByteAndMe var curr = ptr; var end = curr + output.Length; - var currIdx = input.parseStateFirstArgIdx; - ObjectOutputHeader outputHeader = default; try { - if (!input.parseState.TryGetInt(currIdx++, out var start) || - !input.parseState.TryGetInt(currIdx, out var stop)) + if (!input.parseState.TryGetInt(0, out var start) || + !input.parseState.TryGetInt(1, out var stop)) { while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); @@ -741,15 +734,13 @@ private void SortedSetRemoveRangeByScore(ref ObjectInput input, ref SpanByteAndM var curr = ptr; var end = curr + output.Length; - var currIdx = input.parseStateFirstArgIdx; - ObjectOutputHeader outputHeader = default; try { // Read min and max - var minParamBytes = input.parseState.GetArgSliceByRef(currIdx++).ReadOnlySpan; - var maxParamBytes = input.parseState.GetArgSliceByRef(currIdx).ReadOnlySpan; + var minParamBytes = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + var maxParamBytes = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; if (!TryParseParameter(minParamBytes, out var minValue, out var minExclusive) || !TryParseParameter(maxParamBytes, out var maxValue, out var maxExclusive)) @@ -843,10 +834,8 @@ private void SortedSetRemoveOrCountRangeByLex(ref ObjectInput input, byte* outpu // Using minValue for partial execution detection _output->result1 = int.MinValue; - var currIdx = input.parseStateFirstArgIdx; - - var minParamBytes = input.parseState.GetArgSliceByRef(currIdx++).ReadOnlySpan; - var maxParamBytes = input.parseState.GetArgSliceByRef(currIdx).ReadOnlySpan; + var minParamBytes = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; + var maxParamBytes = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; var rem = GetElementsInRangeByLex(minParamBytes, maxParamBytes, false, false, op != SortedSetOperation.ZLEXCOUNT, out int errorCode); @@ -872,13 +861,12 @@ private void SortedSetRank(ref ObjectInput input, ref SpanByteAndMemory output, var curr = ptr; var end = curr + output.Length; - var currIdx = input.parseStateFirstArgIdx; var withScore = input.arg1 == 1; ObjectOutputHeader outputHeader = default; try { - var member = input.parseState.GetArgSliceByRef(currIdx).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); if (!sortedSetDict.TryGetValue(member, out var score)) { diff --git a/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs b/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs index feee1c7e8e..dd0b61a04e 100644 --- a/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs +++ b/libs/server/Objects/SortedSetGeo/SortedSetGeoObjectImpl.cs @@ -59,13 +59,13 @@ private void GeoAdd(ref ObjectInput input, ref SpanByteAndMemory output) var ch = false; var count = input.parseState.Count; - var currTokenIdx = input.parseStateFirstArgIdx; + var currTokenIdx = 0; ObjectOutputHeader _output = default; try { // Read the options - var optsCount = (count - input.parseStateFirstArgIdx) % 3; + var optsCount = count % 3; if (optsCount > 0 && optsCount <= 2) { // Is NX or XX, if not nx then use XX @@ -145,21 +145,16 @@ private void GeoHash(ref ObjectInput input, ref SpanByteAndMemory output) var curr = ptr; var end = curr + output.Length; - var count = input.parseState.Count; - var currTokenIdx = input.parseStateFirstArgIdx; - ObjectOutputHeader _output = default; try { - var tokenCount = input.parseState.Count - input.parseStateFirstArgIdx; - - while (!RespWriteUtils.WriteArrayLength(tokenCount, ref curr, end)) + while (!RespWriteUtils.WriteArrayLength(input.parseState.Count, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - while (currTokenIdx < count) + for (var i = 0; i < input.parseState.Count; i++) { // Read member - var member = input.parseState.GetArgSliceByRef(currTokenIdx++).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); if (sortedSetDict.TryGetValue(member, out var value52Int)) { @@ -193,21 +188,19 @@ private void GeoDistance(ref ObjectInput input, ref SpanByteAndMemory output) var curr = ptr; var end = curr + output.Length; - var currTokenIdx = input.parseStateFirstArgIdx; - ObjectOutputHeader _output = default; try { // Read 1st member - var member1 = input.parseState.GetArgSliceByRef(currTokenIdx++).SpanByte.ToByteArray(); + var member1 = input.parseState.GetArgSliceByRef(0).SpanByte.ToByteArray(); // Read 2nd member - var member2 = input.parseState.GetArgSliceByRef(currTokenIdx++).SpanByte.ToByteArray(); + var member2 = input.parseState.GetArgSliceByRef(1).SpanByte.ToByteArray(); // Read units - var units = input.parseState.Count - currTokenIdx == 0 - ? "M"u8 - : input.parseState.GetArgSliceByRef(currTokenIdx).ReadOnlySpan; + var units = input.parseState.Count > 2 + ? input.parseState.GetArgSliceByRef(2).ReadOnlySpan + : "M"u8; if (sortedSetDict.TryGetValue(member1, out var scoreMember1) && sortedSetDict.TryGetValue(member2, out var scoreMember2)) { @@ -247,20 +240,16 @@ private void GeoPosition(ref ObjectInput input, ref SpanByteAndMemory output) var curr = ptr; var end = curr + output.Length; - var currTokenIdx = input.parseStateFirstArgIdx; - ObjectOutputHeader _output = default; try { - var tokenCount = input.parseState.Count - input.parseStateFirstArgIdx; - - while (!RespWriteUtils.WriteArrayLength(tokenCount, ref curr, end)) + while (!RespWriteUtils.WriteArrayLength(input.parseState.Count, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); - while (currTokenIdx < input.parseState.Count) + for (var i = 0; i < input.parseState.Count; i++) { // read member - var member = input.parseState.GetArgSliceByRef(currTokenIdx++).SpanByte.ToByteArray(); + var member = input.parseState.GetArgSliceByRef(i).SpanByte.ToByteArray(); if (sortedSetDict.TryGetValue(member, out var scoreMember1)) { @@ -302,8 +291,6 @@ private void GeoSearch(ref ObjectInput input, ref SpanByteAndMemory output) var curr = ptr; var end = curr + output.Length; - var currTokenIdx = input.parseStateFirstArgIdx; - ObjectOutputHeader _output = default; try { @@ -316,6 +303,8 @@ private void GeoSearch(ref ObjectInput input, ref SpanByteAndMemory output) ReadOnlySpan errorMessage = default; var argNumError = false; + var currTokenIdx = 0; + // Read the options while (currTokenIdx < input.parseState.Count) { diff --git a/libs/server/Resp/ArrayCommands.cs b/libs/server/Resp/ArrayCommands.cs index 6a6d2ff4d1..c214c4ab33 100644 --- a/libs/server/Resp/ArrayCommands.cs +++ b/libs/server/Resp/ArrayCommands.cs @@ -179,14 +179,13 @@ private bool NetworkMSETNX(ref TGarnetApi storageApi) where TGarnetApi : IGarnetApi { var anyValuesSet = false; - var input = new RawStringInput(RespCommand.SETEXNX, ref parseState); + var input = new RawStringInput(RespCommand.SETEXNX); for (var c = 0; c < parseState.Count; c += 2) { var key = parseState.GetArgSliceByRef(c).SpanByte; - input.parseStateFirstArgIdx = c + 1; - input.parseStateLastArgIdx = input.parseStateFirstArgIdx; + input.parseState = parseState.Slice(c + 1, 1); var status = storageApi.SET_Conditional(ref key, ref input); // Status tells us whether an old image was found during RMW or not @@ -452,7 +451,8 @@ private bool NetworkArrayPING() return NetworkPING(); } - WriteDirectLarge(new ReadOnlySpan(recvBufferPtr + readHead, endReadHead - readHead)); + var message = parseState.GetArgSliceByRef(0).ReadOnlySpan; + WriteDirectLargeRespString(message); return true; } } diff --git a/libs/server/Resp/BasicCommands.cs b/libs/server/Resp/BasicCommands.cs index 48209c052c..bc4f1908d9 100644 --- a/libs/server/Resp/BasicCommands.cs +++ b/libs/server/Resp/BasicCommands.cs @@ -110,7 +110,7 @@ bool NetworkGETEX(ref TGarnetApi storageApi) } var expiry = (tsExpiry.HasValue && tsExpiry.Value.Ticks > 0) ? DateTimeOffset.UtcNow.Ticks + tsExpiry.Value.Ticks : 0; - var input = new RawStringInput(RespCommand.GETEX, ref parseState, 1, -1, expiry); + var input = new RawStringInput(RespCommand.GETEX, ref parseState, startIdx: 1, arg1: expiry); var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); var status = storageApi.GETEX(ref key, ref input, ref o); @@ -334,7 +334,7 @@ private bool NetworkSetRange(ref TGarnetApi storageApi) return true; } - var input = new RawStringInput(RespCommand.SETRANGE, ref parseState, 1); + var input = new RawStringInput(RespCommand.SETRANGE, ref parseState, startIdx: 1); Span outputBuffer = stackalloc byte[NumUtils.MaximumFormatInt64Length]; var output = ArgSlice.FromPinnedSpan(outputBuffer); @@ -361,7 +361,7 @@ private bool NetworkGetRange(ref TGarnetApi storageApi) return true; } - var input = new RawStringInput(RespCommand.GETRANGE, ref parseState, 1); + var input = new RawStringInput(RespCommand.GETRANGE, ref parseState, startIdx: 1); var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); @@ -680,7 +680,7 @@ private bool NetworkSET_Conditional(RespCommand cmd, int expiry, ref ? TimeSpan.FromMilliseconds(expiry).Ticks : TimeSpan.FromSeconds(expiry).Ticks); - var input = new RawStringInput(cmd, ref parseState, 1, -1, inputArg); + var input = new RawStringInput(cmd, ref parseState, startIdx: 1, arg1: inputArg); if (getValue) input.header.SetSetGetFlag(); @@ -801,7 +801,7 @@ private bool NetworkIncrementByFloat(ref TGarnetApi storageApi) Span outputBuffer = stackalloc byte[NumUtils.MaximumFormatDoubleLength + 1]; var output = ArgSlice.FromPinnedSpan(outputBuffer); - var input = new RawStringInput(RespCommand.INCRBYFLOAT, ref parseState, 1); + var input = new RawStringInput(RespCommand.INCRBYFLOAT, ref parseState, startIdx: 1); storageApi.Increment(key, ref input, ref output); var errorFlag = output.Length == NumUtils.MaximumFormatDoubleLength + 1 @@ -834,7 +834,7 @@ private bool NetworkAppend(ref TGarnetApi storageApi) { var sbKey = parseState.GetArgSliceByRef(0).SpanByte; - var input = new RawStringInput(RespCommand.APPEND, ref parseState, 1); + var input = new RawStringInput(RespCommand.APPEND, ref parseState, startIdx: 1); Span outputBuffer = stackalloc byte[NumUtils.MaximumFormatInt64Length]; var output = SpanByteAndMemory.FromPinnedSpan(outputBuffer); @@ -1156,7 +1156,8 @@ private bool NetworkECHO() return AbortWithWrongNumberOfArguments(nameof(RespCommand.ECHO)); } - WriteDirectLarge(new ReadOnlySpan(recvBufferPtr + readHead, endReadHead - readHead)); + var message = parseState.GetArgSliceByRef(0).ReadOnlySpan; + WriteDirectLargeRespString(message); return true; } diff --git a/libs/server/Resp/Bitmap/BitmapCommands.cs b/libs/server/Resp/Bitmap/BitmapCommands.cs index 5f536cf802..787b168354 100644 --- a/libs/server/Resp/Bitmap/BitmapCommands.cs +++ b/libs/server/Resp/Bitmap/BitmapCommands.cs @@ -148,7 +148,7 @@ private bool NetworkStringSetBit(ref TGarnetApi storageApi) return true; } - var input = new RawStringInput(RespCommand.SETBIT, ref parseState, 1); + var input = new RawStringInput(RespCommand.SETBIT, ref parseState, startIdx: 1); var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); var status = storageApi.StringSetBit( @@ -184,7 +184,7 @@ private bool NetworkStringGetBit(ref TGarnetApi storageApi) return true; } - var input = new RawStringInput(RespCommand.GETBIT, ref parseState, 1); + var input = new RawStringInput(RespCommand.GETBIT, ref parseState, startIdx: 1); var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); var status = storageApi.StringGetBit(ref sbKey, ref input, ref o); @@ -226,7 +226,7 @@ private bool NetworkStringBitCount(ref TGarnetApi storageApi) } } - var input = new RawStringInput(RespCommand.BITCOUNT, ref parseState, 1); + var input = new RawStringInput(RespCommand.BITCOUNT, ref parseState, startIdx: 1); var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); @@ -300,7 +300,7 @@ private bool NetworkStringBitPosition(ref TGarnetApi storageApi) } } - var input = new RawStringInput(RespCommand.BITPOS, ref parseState, 1); + var input = new RawStringInput(RespCommand.BITPOS, ref parseState, startIdx: 1); var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); diff --git a/libs/server/Resp/HyperLogLog/HyperLogLog.cs b/libs/server/Resp/HyperLogLog/HyperLogLog.cs index 68891f81a5..3ee7ca037a 100644 --- a/libs/server/Resp/HyperLogLog/HyperLogLog.cs +++ b/libs/server/Resp/HyperLogLog/HyperLogLog.cs @@ -325,7 +325,7 @@ public void InitDense(byte* ptr) /// public int SparseInitialLength(ref RawStringInput input) { - var count = (int)input.arg1; + var count = input.parseState.Count; return SparseInitialLength(count); } @@ -371,7 +371,7 @@ private int SparseRequiredBytes(int cnt) /// public int UpdateGrow(ref RawStringInput input, byte* value) { - var count = (int)input.arg1; + var count = input.parseState.Count; if (IsSparse(value)) { @@ -529,7 +529,7 @@ public bool DenseToDense(byte* srcDenseBlob, byte* dstDenseBlob) /// public bool Update(ref RawStringInput input, byte* value, int valueLen, ref bool updated) { - var count = (int)input.arg1; + var count = input.parseState.Count; if (IsDense(value)) // If blob layout is dense { @@ -602,14 +602,11 @@ private bool UpdateDenseRegister(byte* ptr, ushort idx, byte cntlz) private bool IterateUpdate(ref RawStringInput input, byte* value, bool dense) { var updated = false; - var currTokenIdx = input.parseStateFirstArgIdx; - var elementCount = (int)input.arg1; - while (currTokenIdx < input.parseState.Count && elementCount > 0) + for (var i = 0; i < input.parseState.Count; i++) { - var currElement = input.parseState.GetArgSliceByRef(currTokenIdx++); + var currElement = input.parseState.GetArgSliceByRef(i); var hashValue = (long)HashUtils.MurmurHash2x64A(currElement.ptr, currElement.Length); updated |= (dense ? UpdateDense(value, hashValue) : UpdateSparse(value, hashValue)); - elementCount--; } return updated; } diff --git a/libs/server/Resp/HyperLogLog/HyperLogLogCommands.cs b/libs/server/Resp/HyperLogLog/HyperLogLogCommands.cs index 7273112ad9..74077929a5 100644 --- a/libs/server/Resp/HyperLogLog/HyperLogLogCommands.cs +++ b/libs/server/Resp/HyperLogLog/HyperLogLogCommands.cs @@ -24,8 +24,7 @@ private bool HyperLogLogAdd(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments(nameof(RespCommand.PFADD)); } - var inputArg = 1; // # of elements to add from parseState - var input = new RawStringInput(RespCommand.PFADD, ref parseState, 0, -1, inputArg); + var input = new RawStringInput(RespCommand.PFADD); var output = stackalloc byte[1]; byte pfaddUpdated = 0; @@ -33,8 +32,7 @@ private bool HyperLogLogAdd(ref TGarnetApi storageApi) for (var i = 1; i < parseState.Count; i++) { - input.parseStateFirstArgIdx = i; - input.parseStateLastArgIdx = input.parseStateFirstArgIdx; + input.parseState = parseState.Slice(i, 1); var o = new SpanByteAndMemory(output, 1); storageApi.HyperLogLogAdd(ref key, ref input, ref o); diff --git a/libs/server/Resp/KeyAdminCommands.cs b/libs/server/Resp/KeyAdminCommands.cs index 6ef5b23fe9..9e7f316f46 100644 --- a/libs/server/Resp/KeyAdminCommands.cs +++ b/libs/server/Resp/KeyAdminCommands.cs @@ -233,7 +233,7 @@ private bool NetworkEXPIRE(RespCommand command, ref TGarnetApi stora } } - var input = new RawStringInput(command, ref parseState, 1, -1, (byte)expireOption); + var input = new RawStringInput(command, ref parseState, startIdx: 1, arg1: (byte)expireOption); var status = storageApi.EXPIRE(key, ref input, out var timeoutSet); if (status == GarnetStatus.OK && timeoutSet) @@ -318,7 +318,7 @@ private bool NetworkEXPIREAT(RespCommand command, ref TGarnetApi sto } } - var input = new RawStringInput(command, ref parseState, 1, -1, (byte)expireOption); + var input = new RawStringInput(command, ref parseState, startIdx: 1, arg1: (byte)expireOption); var status = storageApi.EXPIRE(key, ref input, out var timeoutSet); if (status == GarnetStatus.OK && timeoutSet) diff --git a/libs/server/Resp/Objects/HashCommands.cs b/libs/server/Resp/Objects/HashCommands.cs index ace02c1676..818ae9bc9f 100644 --- a/libs/server/Resp/Objects/HashCommands.cs +++ b/libs/server/Resp/Objects/HashCommands.cs @@ -45,7 +45,8 @@ private unsafe bool HashSet(RespCommand command, ref TGarnetApi stor // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = hop }; - var input = new ObjectInput(header, ref parseState, 1); + + var input = new ObjectInput(header, ref parseState, startIdx: 1); var status = storageApi.HashSet(keyBytes, ref input, out var output); @@ -90,7 +91,7 @@ private bool HashGet(RespCommand command, ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HGET }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); // Prepare GarnetObjectStore output var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; @@ -177,7 +178,8 @@ private bool HashGetMultiple(RespCommand command, ref TGarnetApi sto // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HMGET }; - var input = new ObjectInput(header, ref parseState, 1); + + var input = new ObjectInput(header, ref parseState, startIdx: 1); // Prepare GarnetObjectStore output var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; @@ -354,7 +356,7 @@ private unsafe bool HashStrLength(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HSTRLEN }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); var status = storageApi.HashStrLength(keyBytes, ref input, out var output); @@ -397,7 +399,7 @@ private unsafe bool HashDelete(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HDEL }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); var status = storageApi.HashDelete(keyBytes, ref input, out var output); @@ -438,7 +440,7 @@ private unsafe bool HashExists(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = HashOperation.HEXISTS }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); var status = storageApi.HashExists(keyBytes, ref input, out var output); @@ -548,7 +550,7 @@ private unsafe bool HashIncrement(RespCommand command, ref TGarnetAp // Prepare input var header = new RespInputHeader(GarnetObjectType.Hash) { HashOp = op }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); // Prepare GarnetObjectStore output var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; diff --git a/libs/server/Resp/Objects/ListCommands.cs b/libs/server/Resp/Objects/ListCommands.cs index f13096050d..62be648670 100644 --- a/libs/server/Resp/Objects/ListCommands.cs +++ b/libs/server/Resp/Objects/ListCommands.cs @@ -41,7 +41,7 @@ private unsafe bool ListPush(RespCommand command, ref TGarnetApi sto // Prepare input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = lop }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); var status = command == RespCommand.LPUSH || command == RespCommand.LPUSHX ? storageApi.ListLeftPush(keyBytes, ref input, out var output) @@ -153,7 +153,7 @@ private unsafe bool ListPosition(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LPOS }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); // Prepare GarnetObjectStore output var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; @@ -615,7 +615,7 @@ private bool ListInsert(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LINSERT }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); var statusOp = storageApi.ListInsert(keyBytes, ref input, out var output); @@ -671,7 +671,7 @@ private bool ListRemove(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LREM }; - var input = new ObjectInput(header, ref parseState, 2, -1, nCount); + var input = new ObjectInput(header, ref parseState, startIdx: 2, arg1: nCount); var statusOp = storageApi.ListRemove(keyBytes, ref input, out var output); @@ -844,7 +844,7 @@ public bool ListSet(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.List) { ListOp = ListOperation.LSET }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); // Prepare GarnetObjectStore output var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; diff --git a/libs/server/Resp/Objects/SetCommands.cs b/libs/server/Resp/Objects/SetCommands.cs index 619c86f4d4..aaae5ace61 100644 --- a/libs/server/Resp/Objects/SetCommands.cs +++ b/libs/server/Resp/Objects/SetCommands.cs @@ -35,7 +35,7 @@ private unsafe bool SetAdd(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SADD }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); var status = storageApi.SetAdd(keyBytes, ref input, out var output); @@ -261,7 +261,7 @@ private unsafe bool SetRemove(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = SetOperation.SREM }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); var status = storageApi.SetRemove(keyBytes, ref input, out var output); @@ -402,7 +402,7 @@ private unsafe bool SetIsMember(RespCommand cmd, ref TGarnetApi stor // Prepare input var header = new RespInputHeader(GarnetObjectType.Set) { SetOp = isSingle ? SetOperation.SISMEMBER : SetOperation.SMISMEMBER }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); // Prepare GarnetObjectStore output var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; diff --git a/libs/server/Resp/Objects/SharedObjectCommands.cs b/libs/server/Resp/Objects/SharedObjectCommands.cs index ae78e62af3..07b32fe245 100644 --- a/libs/server/Resp/Objects/SharedObjectCommands.cs +++ b/libs/server/Resp/Objects/SharedObjectCommands.cs @@ -47,8 +47,8 @@ private unsafe bool ObjectScan(GarnetObjectType objectType, ref TGar } var header = new RespInputHeader(objectType); - var input = new ObjectInput(header, ref parseState, 2, -1, cursorValue, - storeWrapper.serverOptions.ObjectScanCountLimit); + var input = new ObjectInput(header, ref parseState, startIdx: 2, arg1: cursorValue, + arg2: storeWrapper.serverOptions.ObjectScanCountLimit); switch (objectType) { diff --git a/libs/server/Resp/Objects/SortedSetCommands.cs b/libs/server/Resp/Objects/SortedSetCommands.cs index c48f5bccef..93d7b81b40 100644 --- a/libs/server/Resp/Objects/SortedSetCommands.cs +++ b/libs/server/Resp/Objects/SortedSetCommands.cs @@ -32,7 +32,7 @@ private unsafe bool SortedSetAdd(ref TGarnetApi storageApi) var keyBytes = sbKey.ToByteArray(); var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZADD }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; @@ -72,7 +72,7 @@ private unsafe bool SortedSetRemove(ref TGarnetApi storageApi) var keyBytes = sbKey.ToByteArray(); var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZREM }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); var status = storageApi.SortedSetRemove(keyBytes, ref input, out var rmwOutput); @@ -169,7 +169,7 @@ private unsafe bool SortedSetRange(RespCommand command, ref TGarnetA }; var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = op }; - var input = new ObjectInput(header, ref parseState, 1, -1, respProtocolVersion); + var input = new ObjectInput(header, ref parseState, startIdx: 1, arg1: respProtocolVersion); var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; @@ -215,7 +215,7 @@ private unsafe bool SortedSetScore(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZSCORE }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); // Prepare GarnetObjectStore output var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; @@ -262,7 +262,7 @@ private unsafe bool SortedSetScores(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZMSCORE }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); // Prepare GarnetObjectStore output var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; @@ -376,7 +376,7 @@ private unsafe bool SortedSetCount(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZCOUNT }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); // Prepare output var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(SpanByte.FromPinnedPointer(dcurr, (int)(dend - dcurr))) }; @@ -434,7 +434,7 @@ private unsafe bool SortedSetLengthByValue(RespCommand command, ref // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = op }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); var status = op == SortedSetOperation.ZREMRANGEBYLEX ? storageApi.SortedSetRemoveRangeByLex(keyBytes, ref input, out var output) : @@ -491,7 +491,7 @@ private unsafe bool SortedSetIncrement(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.ZINCRBY }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); // Prepare GarnetObjectStore output var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; @@ -560,7 +560,7 @@ private unsafe bool SortedSetRank(RespCommand command, ref TGarnetAp // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = op }; - var input = new ObjectInput(header, ref parseState, 1, -1, includeWithScore ? 1 : 0); + var input = new ObjectInput(header, ref parseState, startIdx: 1, arg1: includeWithScore ? 1 : 0); // Prepare GarnetObjectStore output var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; @@ -616,7 +616,7 @@ private unsafe bool SortedSetRemoveRange(RespCommand command, ref TG // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = op }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); // Prepare GarnetObjectStore output var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; diff --git a/libs/server/Resp/Objects/SortedSetGeoCommands.cs b/libs/server/Resp/Objects/SortedSetGeoCommands.cs index 1d36fa6001..4da54e0d10 100644 --- a/libs/server/Resp/Objects/SortedSetGeoCommands.cs +++ b/libs/server/Resp/Objects/SortedSetGeoCommands.cs @@ -31,7 +31,7 @@ private unsafe bool GeoAdd(ref TGarnetApi storageApi) // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = SortedSetOperation.GEOADD }; - var input = new ObjectInput(header, ref parseState, 1); + var input = new ObjectInput(header, ref parseState, startIdx: 1); var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; @@ -104,7 +104,8 @@ private unsafe bool GeoCommands(RespCommand command, ref TGarnetApi // Prepare input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = op }; - var input = new ObjectInput(header, ref parseState, 1); + + var input = new ObjectInput(header, ref parseState, startIdx: 1); var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)) }; diff --git a/libs/server/Resp/Parser/SessionParseState.cs b/libs/server/Resp/Parser/SessionParseState.cs index ee916fc860..59f72dd7f5 100644 --- a/libs/server/Resp/Parser/SessionParseState.cs +++ b/libs/server/Resp/Parser/SessionParseState.cs @@ -22,33 +22,47 @@ public unsafe struct SessionParseState const int MinParams = 5; // 5 * 20 = 60; around one cache line of 64 bytes /// - /// Count of arguments for the command + /// Count of accessible arguments for the command /// public int Count; /// - /// Pointer to buffer + /// Pointer to accessible buffer /// ArgSlice* bufferPtr; /// - /// Arguments buffer + /// Count of arguments in the original buffer /// - ArgSlice[] buffer; + int rootCount; + + /// + /// Arguments original buffer + /// + ArgSlice[] rootBuffer; /// /// Get a Span of the parsed parameters in the form an ArgSlice /// public ReadOnlySpan Parameters => new(bufferPtr, Count); + private SessionParseState(ref ArgSlice[] rootBuffer, int rootCount, ref ArgSlice* bufferPtr, int count) : this() + { + this.rootBuffer = rootBuffer; + this.rootCount = rootCount; + this.bufferPtr = bufferPtr; + this.Count = count; + } + /// /// Initialize the parse state at the start of a session /// public void Initialize() { Count = 0; - buffer = GC.AllocateArray(MinParams, true); - bufferPtr = (ArgSlice*)Unsafe.AsPointer(ref buffer[0]); + rootCount = 0; + rootBuffer = GC.AllocateArray(MinParams, true); + bufferPtr = (ArgSlice*)Unsafe.AsPointer(ref rootBuffer[0]); } /// @@ -58,13 +72,14 @@ public void Initialize() [MethodImpl(MethodImplOptions.AggressiveInlining)] public void Initialize(int count) { - this.Count = count; + Count = count; + rootCount = count; - if (buffer != null && (count <= MinParams || count <= buffer.Length)) + if (rootBuffer != null && (count <= MinParams || count <= rootBuffer.Length)) return; - buffer = GC.AllocateArray(count <= MinParams ? MinParams : count, true); - bufferPtr = (ArgSlice*)Unsafe.AsPointer(ref buffer[0]); + rootBuffer = GC.AllocateArray(count <= MinParams ? MinParams : count, true); + bufferPtr = (ArgSlice*)Unsafe.AsPointer(ref rootBuffer[0]); } /// @@ -76,7 +91,7 @@ public void InitializeWithArgument(ArgSlice arg) { Initialize(1); - buffer[0] = arg; + *bufferPtr = arg; } /// @@ -89,8 +104,8 @@ public void InitializeWithArguments(ArgSlice arg1, ArgSlice arg2) { Initialize(2); - buffer[0] = arg1; - buffer[1] = arg2; + *bufferPtr = arg1; + *(bufferPtr + 1) = arg2; } /// @@ -104,9 +119,9 @@ public void InitializeWithArguments(ArgSlice arg1, ArgSlice arg2, ArgSlice arg3) { Initialize(3); - buffer[0] = arg1; - buffer[1] = arg2; - buffer[2] = arg3; + *bufferPtr = arg1; + *(bufferPtr + 1) = arg2; + *(bufferPtr + 2) = arg3; } /// @@ -121,10 +136,10 @@ public void InitializeWithArguments(ArgSlice arg1, ArgSlice arg2, ArgSlice arg3, { Initialize(4); - buffer[0] = arg1; - buffer[1] = arg2; - buffer[2] = arg3; - buffer[3] = arg4; + *bufferPtr = arg1; + *(bufferPtr + 1) = arg2; + *(bufferPtr + 2) = arg3; + *(bufferPtr + 3) = arg4; } /// @@ -140,11 +155,11 @@ public void InitializeWithArguments(ArgSlice arg1, ArgSlice arg2, ArgSlice arg3, { Initialize(5); - buffer[0] = arg1; - buffer[1] = arg2; - buffer[2] = arg3; - buffer[3] = arg4; - buffer[4] = arg5; + *bufferPtr = arg1; + *(bufferPtr + 1) = arg2; + *(bufferPtr + 2) = arg3; + *(bufferPtr + 3) = arg4; + *(bufferPtr + 4) = arg5; } /// @@ -158,10 +173,37 @@ public void InitializeWithArguments(ArgSlice[] args) for (var i = 0; i < args.Length; i++) { - buffer[i] = args[i]; + *(bufferPtr + i) = args[i]; } } + /// + /// Limit access to the argument buffer to start at a specified index. + /// + /// Offset value to the underlying buffer + public SessionParseState Slice(int idxOffset) + { + Debug.Assert(idxOffset - 1 < rootCount); + + var count = rootCount - idxOffset; + var offsetBuffer = bufferPtr + idxOffset; + return new SessionParseState(ref rootBuffer, rootCount, ref offsetBuffer, count); + } + + /// + /// Limit access to the argument buffer to start at a specified index + /// and end after a specified number of arguments. + /// + /// Offset value to the underlying buffer + /// Argument count + public SessionParseState Slice(int idxOffset, int count) + { + Debug.Assert(idxOffset + count - 1 < rootCount); + + var offsetBuffer = bufferPtr + idxOffset; + return new SessionParseState(ref rootBuffer, rootCount, ref offsetBuffer, count); + } + /// /// Set argument at a specific index /// @@ -170,7 +212,7 @@ public void InitializeWithArguments(ArgSlice[] args) public void SetArgument(int i, ArgSlice arg) { Debug.Assert(i < Count); - buffer[i] = arg; + *(bufferPtr + i) = arg; } /// @@ -183,31 +225,21 @@ public void SetArguments(int i, params ArgSlice[] args) Debug.Assert(i + args.Length - 1 < Count); for (var j = 0; j < args.Length; j++) { - buffer[i + j] = args[j]; + *(bufferPtr + i + j) = args[j]; } } /// - /// Get serialized length of parse state when arguments are only - /// serialized starting at a specified index + /// Get serialized length of parse state /// - /// First index from which arguments are serialized - /// Last index of arguments to serialize /// The serialized length - public int GetSerializedLength(int firstIdx, int lastIdx) + public int GetSerializedLength() { var serializedLength = sizeof(int); - var argCount = Count == 0 ? 0 : (lastIdx == -1 ? Count : lastIdx + 1) - firstIdx; - - if (argCount > 0) + for (var i = 0; i < Count; i++) { - Debug.Assert(firstIdx + argCount <= Count); - - for (var i = 0; i < argCount; i++) - { - serializedLength += buffer[firstIdx + i].SpanByte.TotalSize; - } + serializedLength += (*(bufferPtr + i)).SpanByte.TotalSize; } return serializedLength; @@ -218,32 +250,22 @@ public int GetSerializedLength(int firstIdx, int lastIdx) /// when arguments are only serialized starting at a specified index /// /// The memory buffer to serialize into (of size at least SerializedLength(firstIdx) bytes) - /// First index from which arguments are serialized - /// Last index of arguments to serialize /// Length of buffer to serialize into. /// Total serialized bytes - public int CopyTo(byte* dest, int firstIdx, int lastIdx, int length) + public int CopyTo(byte* dest, int length) { - Debug.Assert(length >= this.GetSerializedLength(firstIdx, lastIdx)); - var curr = dest; // Serialize argument count - var argCount = Count == 0 ? 0 : (lastIdx == -1 ? Count : lastIdx + 1) - firstIdx; - *(int*)curr = argCount; + *(int*)curr = Count; curr += sizeof(int); // Serialize arguments - if (argCount > 0) + for (var i = 0; i < Count; i++) { - Debug.Assert(firstIdx + argCount <= Count); - - for (var i = 0; i < argCount; i++) - { - var sbParam = buffer[firstIdx + i].SpanByte; - sbParam.CopyTo(curr); - curr += sbParam.TotalSize; - } + var sbParam = (*(bufferPtr + i)).SpanByte; + sbParam.CopyTo(curr); + curr += sbParam.TotalSize; } return (int)(dest - curr); @@ -266,7 +288,7 @@ public unsafe int DeserializeFrom(byte* src) for (var i = 0; i < argCount; i++) { ref var sbArgument = ref Unsafe.AsRef(curr); - buffer[i] = new ArgSlice(ref sbArgument); + *(bufferPtr + i) = new ArgSlice(ref sbArgument); curr += sbArgument.TotalSize; } diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index ef7d28d8ac..f0b44d0026 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -1003,6 +1003,18 @@ private void SendAndReset(IMemoryOwner memory, int length) memory.Dispose(); } + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void WriteDirectLargeRespString(ReadOnlySpan message) + { + while (!RespWriteUtils.WriteBulkStringLength(message, ref dcurr, dend)) + SendAndReset(); + + WriteDirectLarge(message); + + while (!RespWriteUtils.WriteNewLine(ref dcurr, dend)) + SendAndReset(); + } + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void WriteDirectLarge(ReadOnlySpan src) { diff --git a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs index 50bf90392a..3bbe2f1fab 100644 --- a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs +++ b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs @@ -134,7 +134,7 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB break; case RespCommand.GETBIT: - var offset = input.parseState.GetLong(input.parseStateFirstArgIdx); + var offset = input.parseState.GetLong(0); var oldValSet = BitmapManager.GetBit(offset, value.ToPointer(), value.Length); if (oldValSet == 0) CopyDefaultResp(CmdStrings.RESP_RETURN_VAL_0, ref dst); @@ -143,19 +143,18 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB break; case RespCommand.BITCOUNT: - var currTokenIdx = input.parseStateFirstArgIdx; var bcStartOffset = 0; var bcEndOffset = -1; byte bcOffsetType = 0x0; - if (currTokenIdx + 1 < input.parseState.Count) + if (input.parseState.Count > 1) { - bcStartOffset = input.parseState.GetInt(currTokenIdx++); - bcEndOffset = input.parseState.GetInt(currTokenIdx++); + bcStartOffset = input.parseState.GetInt(0); + bcEndOffset = input.parseState.GetInt(1); - if (currTokenIdx < input.parseState.Count) + if (input.parseState.Count > 2) { - var spanOffsetType = input.parseState.GetArgSliceByRef(currTokenIdx).ReadOnlySpan; + var spanOffsetType = input.parseState.GetArgSliceByRef(2).ReadOnlySpan; bcOffsetType = spanOffsetType.EqualsUpperCaseSpanIgnoringCase("BIT"u8) ? (byte)0x1 : (byte)0x0; } } @@ -165,20 +164,19 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB break; case RespCommand.BITPOS: - currTokenIdx = input.parseStateFirstArgIdx; - var bpSetVal = (byte)(input.parseState.GetArgSliceByRef(currTokenIdx++).ReadOnlySpan[0] - '0'); + var bpSetVal = (byte)(input.parseState.GetArgSliceByRef(0).ReadOnlySpan[0] - '0'); var bpStartOffset = 0; var bpEndOffset = -1; byte bpOffsetType = 0x0; - if (input.parseState.Count - currTokenIdx > 0) + if (input.parseState.Count > 1) { - bpStartOffset = input.parseState.GetInt(currTokenIdx++); - if (input.parseState.Count - currTokenIdx > 0) + bpStartOffset = input.parseState.GetInt(1); + if (input.parseState.Count > 2) { - bpEndOffset = input.parseState.GetInt(currTokenIdx++); - if (input.parseState.Count - currTokenIdx > 0) + bpEndOffset = input.parseState.GetInt(2); + if (input.parseState.Count > 3) { - var sbOffsetType = input.parseState.GetArgSliceByRef(currTokenIdx).ReadOnlySpan; + var sbOffsetType = input.parseState.GetArgSliceByRef(3).ReadOnlySpan; bpOffsetType = sbOffsetType.EqualsUpperCaseSpanIgnoringCase("BIT"u8) ? (byte)0x1 : (byte)0x0; @@ -239,8 +237,8 @@ void CopyRespToWithInput(ref RawStringInput input, ref SpanByte value, ref SpanB case RespCommand.GETRANGE: var len = value.LengthWithoutMetadata; - var start = input.parseState.GetInt(input.parseStateFirstArgIdx); - var end = input.parseState.GetInt(input.parseStateFirstArgIdx + 1); + var start = input.parseState.GetInt(0); + var end = input.parseState.GetInt(1); (start, end) = NormalizeRange(start, end, len); CopyRespTo(ref value, ref dst, start, end); @@ -712,7 +710,7 @@ void WriteLogDelete(ref SpanByte key, long version, int sessionID) BitFieldCmdArgs GetBitFieldArguments(ref RawStringInput input) { - var currTokenIdx = input.parseStateFirstArgIdx; + var currTokenIdx = 0; var cmd = input.parseState.GetEnum(currTokenIdx++, true); var encodingArg = input.parseState.GetString(currTokenIdx++); var offsetArg = input.parseState.GetString(currTokenIdx++); diff --git a/libs/server/Storage/Functions/MainStore/RMWMethods.cs b/libs/server/Storage/Functions/MainStore/RMWMethods.cs index 6870eba54f..c1728e1d6f 100644 --- a/libs/server/Storage/Functions/MainStore/RMWMethods.cs +++ b/libs/server/Storage/Functions/MainStore/RMWMethods.cs @@ -61,7 +61,7 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB case RespCommand.PFMERGE: //srcHLL offset: [hll allocated size = 4 byte] + [hll data structure] //memcpy + 4 (skip len size) - var sbSrcHLL = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).SpanByte; + var sbSrcHLL = input.parseState.GetArgSliceByRef(0).SpanByte; var length = sbSrcHLL.Length; var srcHLL = sbSrcHLL.ToPointer(); var dstHLL = value.ToPointer(); @@ -74,7 +74,7 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB case RespCommand.SET: case RespCommand.SETEXNX: // Copy input to value - var newInputValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).ReadOnlySpan; + var newInputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; var metadataSize = input.arg1 == 0 ? 0 : sizeof(long); value.UnmarkExtraMetadata(); value.ShrinkSerializedLength(newInputValue.Length + metadataSize); @@ -84,7 +84,7 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB case RespCommand.SETKEEPTTL: // Copy input to value, retain metadata in value - var setValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).ReadOnlySpan; + var setValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; value.ShrinkSerializedLength(value.MetadataSize + setValue.Length); setValue.CopyTo(value.AsSpan()); break; @@ -101,9 +101,8 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB throw new Exception(); case RespCommand.SETBIT: - var currTokenIdx = input.parseStateFirstArgIdx; - var bOffset = input.parseState.GetLong(currTokenIdx++); - var bSetVal = (byte)(input.parseState.GetArgSliceByRef(currTokenIdx).ReadOnlySpan[0] - '0'); + var bOffset = input.parseState.GetLong(0); + var bSetVal = (byte)(input.parseState.GetArgSliceByRef(1).ReadOnlySpan[0] - '0'); value.UnmarkExtraMetadata(); value.ShrinkSerializedLength(BitmapManager.Length(bOffset)); @@ -125,15 +124,15 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB break; case RespCommand.SETRANGE: - var offset = input.parseState.GetInt(input.parseStateFirstArgIdx); - var newValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx + 1).ReadOnlySpan; + var offset = input.parseState.GetInt(0); + var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; newValue.CopyTo(value.AsSpan().Slice(offset)); CopyValueLengthToOutput(ref value, ref output); break; case RespCommand.APPEND: - var appendValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx); + var appendValue = input.parseState.GetArgSliceByRef(0); // Copy value to be appended to the newly allocated value buffer appendValue.ReadOnlySpan.CopyTo(value.AsSpan()); @@ -169,7 +168,7 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB case RespCommand.INCRBYFLOAT: value.UnmarkExtraMetadata(); // Check if input contains a valid number - if (!input.parseState.TryGetDouble(input.parseStateFirstArgIdx, out var incrByFloat)) + if (!input.parseState.TryGetDouble(0, out var incrByFloat)) { output.SpanByte.AsSpan()[0] = (byte)OperationError.INVALID_TYPE; return true; @@ -203,7 +202,7 @@ public bool InitialUpdater(ref SpanByte key, ref RawStringInput input, ref SpanB } // Copy input to value - var inputValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx); + var inputValue = input.parseState.GetArgSliceByRef(0); value.ShrinkSerializedLength(inputValue.Length); value.ExtraMetadata = input.arg1; inputValue.ReadOnlySpan.CopyTo(value.AsSpan()); @@ -266,7 +265,7 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re case RespCommand.SET: case RespCommand.SETEXXX: - var setValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx); + var setValue = input.parseState.GetArgSliceByRef(0); // Need CU if no space for new value var metadataSize = input.arg1 == 0 ? 0 : sizeof(long); @@ -293,7 +292,7 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re case RespCommand.SETKEEPTTLXX: case RespCommand.SETKEEPTTL: - setValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx); + setValue = input.parseState.GetArgSliceByRef(0); // Need CU if no space for new value if (setValue.Length + value.MetadataSize > value.Length) return false; @@ -317,7 +316,7 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re case RespCommand.EXPIRE: var expiryExists = value.MetadataSize > 0; - var expiryValue = input.parseState.GetLong(input.parseStateFirstArgIdx); + var expiryValue = input.parseState.GetLong(0); var tsExpiry = input.header.cmd == RespCommand.EXPIRE ? TimeSpan.FromSeconds(expiryValue) : TimeSpan.FromMilliseconds(expiryValue); @@ -330,7 +329,7 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re case RespCommand.EXPIREAT: expiryExists = value.MetadataSize > 0; - var expiryTimestamp = input.parseState.GetLong(input.parseStateFirstArgIdx); + var expiryTimestamp = input.parseState.GetLong(0); expiryTicks = input.header.cmd == RespCommand.PEXPIREAT ? ConvertUtils.UnixTimestampInMillisecondsToTicks(expiryTimestamp) : ConvertUtils.UnixTimestampInSecondsToTicks(expiryTimestamp); @@ -367,7 +366,7 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re case RespCommand.INCRBYFLOAT: // Check if input contains a valid number - if (!input.parseState.TryGetDouble(input.parseStateFirstArgIdx, out var incrByFloat)) + if (!input.parseState.TryGetDouble(0, out var incrByFloat)) { output.SpanByte.AsSpan()[0] = (byte)OperationError.INVALID_TYPE; return true; @@ -376,9 +375,8 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re case RespCommand.SETBIT: var v = value.ToPointer(); - var currTokenIdx = input.parseStateFirstArgIdx; - var bOffset = input.parseState.GetLong(currTokenIdx++); - var bSetVal = (byte)(input.parseState.GetArgSliceByRef(currTokenIdx).ReadOnlySpan[0] - '0'); + var bOffset = input.parseState.GetLong(0); + var bSetVal = (byte)(input.parseState.GetArgSliceByRef(1).ReadOnlySpan[0] - '0'); if (!BitmapManager.IsLargeEnough(value.Length, bOffset)) return false; @@ -432,7 +430,7 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re case RespCommand.PFMERGE: //srcHLL offset: [hll allocated size = 4 byte] + [hll data structure] //memcpy +4 (skip len size) - var srcHLL = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).SpanByte.ToPointer(); + var srcHLL = input.parseState.GetArgSliceByRef(0).SpanByte.ToPointer(); var dstHLL = value.ToPointer(); if (!HyperLogLog.DefaultHLL.IsValidHYLL(dstHLL, value.Length)) @@ -446,8 +444,8 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re rmwInfo.SetUsedValueLength(ref recordInfo, ref value, value.TotalSize); return HyperLogLog.DefaultHLL.TryMerge(srcHLL, dstHLL, value.Length); case RespCommand.SETRANGE: - var offset = input.parseState.GetInt(input.parseStateFirstArgIdx); - var newValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx + 1).ReadOnlySpan; + var offset = input.parseState.GetInt(0); + var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; if (newValue.Length + offset > value.LengthWithoutMetadata) return false; @@ -476,9 +474,9 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re return EvaluateExpireInPlace(ExpireOption.None, expiryExists: value.MetadataSize > 0, newExpiry, ref value, ref _output); } - if (input.parseState.Count - input.parseStateFirstArgIdx > 0) + if (input.parseState.Count > 0) { - var persist = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).ReadOnlySpan + var persist = input.parseState.GetArgSliceByRef(0).ReadOnlySpan .EqualsUpperCaseSpanIgnoringCase(CmdStrings.PERSIST); if (persist) // Persist the key @@ -496,7 +494,7 @@ private bool InPlaceUpdaterWorker(ref SpanByte key, ref RawStringInput input, re case RespCommand.APPEND: // If nothing to append, can avoid copy update. - var appendSize = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).Length; + var appendSize = input.parseState.GetArgSliceByRef(0).Length; if (appendSize == 0) { @@ -614,7 +612,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte } // Copy input to value - var newInputValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).ReadOnlySpan; + var newInputValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; var metadataSize = input.arg1 == 0 ? 0 : sizeof(long); Debug.Assert(newInputValue.Length + metadataSize == newValue.Length); @@ -625,7 +623,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte case RespCommand.SETKEEPTTLXX: case RespCommand.SETKEEPTTL: - var setValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).ReadOnlySpan; + var setValue = input.parseState.GetArgSliceByRef(0).ReadOnlySpan; Debug.Assert(oldValue.MetadataSize + setValue.Length == newValue.Length); // Check if SetGet flag is set @@ -644,7 +642,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte case RespCommand.PEXPIRE: var expiryExists = oldValue.MetadataSize > 0; - var expiryValue = input.parseState.GetLong(input.parseStateFirstArgIdx); + var expiryValue = input.parseState.GetLong(0); var tsExpiry = input.header.cmd == RespCommand.EXPIRE ? TimeSpan.FromSeconds(expiryValue) : TimeSpan.FromMilliseconds(expiryValue); @@ -658,7 +656,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte case RespCommand.EXPIREAT: expiryExists = oldValue.MetadataSize > 0; - var expiryTimestamp = input.parseState.GetLong(input.parseStateFirstArgIdx); + var expiryTimestamp = input.parseState.GetLong(0); expiryTicks = input.header.cmd == RespCommand.PEXPIREAT ? ConvertUtils.UnixTimestampInMillisecondsToTicks(expiryTimestamp) : ConvertUtils.UnixTimestampInSecondsToTicks(expiryTimestamp); @@ -698,7 +696,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte case RespCommand.INCRBYFLOAT: // Check if input contains a valid number - if (!input.parseState.TryGetDouble(input.parseStateFirstArgIdx, out var incrByFloat)) + if (!input.parseState.TryGetDouble(0, out var incrByFloat)) { // Move to tail of the log oldValue.CopyTo(ref newValue); @@ -708,9 +706,8 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte break; case RespCommand.SETBIT: - var currTokenIdx = input.parseStateFirstArgIdx; - var bOffset = input.parseState.GetLong(currTokenIdx++); - var bSetVal = (byte)(input.parseState.GetArgSliceByRef(currTokenIdx).ReadOnlySpan[0] - '0'); + var bOffset = input.parseState.GetLong(0); + var bSetVal = (byte)(input.parseState.GetArgSliceByRef(1).ReadOnlySpan[0] - '0'); Buffer.MemoryCopy(oldValue.ToPointer(), newValue.ToPointer(), newValue.Length, oldValue.Length); var oldValSet = BitmapManager.UpdateBitmap(newValue.ToPointer(), bOffset, bSetVal); if (oldValSet == 0) @@ -747,7 +744,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte case RespCommand.PFMERGE: //srcA offset: [hll allocated size = 4 byte] + [hll data structure] //memcpy +4 (skip len size) - var srcHLLPtr = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).SpanByte.ToPointer(); // HLL merging from + var srcHLLPtr = input.parseState.GetArgSliceByRef(0).SpanByte.ToPointer(); // HLL merging from var oldDstHLLPtr = oldValue.ToPointer(); // original HLL merging to (too small to hold its data plus srcA) var newDstHLLPtr = newValue.ToPointer(); // new HLL merging to (large enough to hold srcA and srcB @@ -755,10 +752,10 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte break; case RespCommand.SETRANGE: - var offset = input.parseState.GetInt(input.parseStateFirstArgIdx); + var offset = input.parseState.GetInt(0); oldValue.CopyTo(ref newValue); - newInputValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx + 1).ReadOnlySpan; + newInputValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; newInputValue.CopyTo(newValue.AsSpan().Slice(offset)); CopyValueLengthToOutput(ref newValue, ref output); @@ -785,9 +782,9 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); - if (input.parseState.Count - input.parseStateFirstArgIdx > 0) + if (input.parseState.Count > 0) { - var persist = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).ReadOnlySpan + var persist = input.parseState.GetArgSliceByRef(0).ReadOnlySpan .EqualsUpperCaseSpanIgnoringCase(CmdStrings.PERSIST); if (persist) // Persist the key @@ -803,7 +800,7 @@ public bool CopyUpdater(ref SpanByte key, ref RawStringInput input, ref SpanByte // Copy any existing value with metadata to thew new value oldValue.CopyTo(ref newValue); - var appendValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx); + var appendValue = input.parseState.GetArgSliceByRef(0); // Append the new value with the client input at the end of the old data appendValue.ReadOnlySpan.CopyTo(newValue.AsSpan().Slice(oldValue.LengthWithoutMetadata)); diff --git a/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs b/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs index e17917a546..b0b3803465 100644 --- a/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs +++ b/libs/server/Storage/Functions/MainStore/VarLenInputMethods.cs @@ -73,7 +73,7 @@ public int GetRMWInitialValueLength(ref RawStringInput input) switch (cmd) { case RespCommand.SETBIT: - var bOffset = input.parseState.GetLong(input.parseStateFirstArgIdx); + var bOffset = input.parseState.GetLong(0); return sizeof(int) + BitmapManager.Length(bOffset); case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); @@ -81,15 +81,15 @@ public int GetRMWInitialValueLength(ref RawStringInput input) case RespCommand.PFADD: return sizeof(int) + HyperLogLog.DefaultHLL.SparseInitialLength(ref input); case RespCommand.PFMERGE: - var length = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).SpanByte.Length; + var length = input.parseState.GetArgSliceByRef(0).SpanByte.Length; return sizeof(int) + length; case RespCommand.SETRANGE: - var offset = input.parseState.GetInt(input.parseStateFirstArgIdx); - var newValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx + 1).ReadOnlySpan; + var offset = input.parseState.GetInt(0); + var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; return sizeof(int) + newValue.Length + offset; case RespCommand.APPEND: - var valueLength = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).Length; + var valueLength = input.parseState.GetArgSliceByRef(0).Length; return sizeof(int) + valueLength; case RespCommand.INCR: @@ -111,7 +111,7 @@ public int GetRMWInitialValueLength(ref RawStringInput input) return sizeof(int) + ndigits + (fNeg ? 1 : 0); case RespCommand.INCRBYFLOAT: - if (!input.parseState.TryGetDouble(input.parseStateFirstArgIdx, out var incrByFloat)) + if (!input.parseState.TryGetDouble(0, out var incrByFloat)) return sizeof(int); ndigits = NumUtils.NumOfCharInDouble(incrByFloat, out var _, out var _, out var _); @@ -132,7 +132,7 @@ public int GetRMWInitialValueLength(ref RawStringInput input) return sizeof(int) + metadataSize + functions.GetInitialLength(ref input); } - return sizeof(int) + input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).ReadOnlySpan.Length + + return sizeof(int) + input.parseState.GetArgSliceByRef(0).ReadOnlySpan.Length + (input.arg1 == 0 ? 0 : sizeof(long)); } } @@ -172,7 +172,7 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) return sizeof(int) + ndigits + t.MetadataSize; case RespCommand.INCRBYFLOAT: // We don't need to TryGetDouble here because InPlaceUpdater will raise an error before we reach this point - var incrByFloat = input.parseState.GetDouble(input.parseStateFirstArgIdx); + var incrByFloat = input.parseState.GetDouble(0); NumUtils.TryBytesToDouble(t.AsSpan(), out var currVal); var nextVal = currVal + incrByFloat; @@ -181,7 +181,7 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) return sizeof(int) + ndigits + t.MetadataSize; case RespCommand.SETBIT: - var bOffset = input.parseState.GetLong(input.parseStateFirstArgIdx); + var bOffset = input.parseState.GetLong(0); return sizeof(int) + BitmapManager.NewBlockAllocLength(t.Length, bOffset); case RespCommand.BITFIELD: var bitFieldArgs = GetBitFieldArguments(ref input); @@ -194,14 +194,14 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) case RespCommand.PFMERGE: length = sizeof(int); - var srcHLL = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).SpanByte.ToPointer(); + var srcHLL = input.parseState.GetArgSliceByRef(0).SpanByte.ToPointer(); var dstHLL = t.ToPointer(); length += HyperLogLog.DefaultHLL.MergeGrow(srcHLL, dstHLL); return length + t.MetadataSize; case RespCommand.SETKEEPTTLXX: case RespCommand.SETKEEPTTL: - var setValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx); + var setValue = input.parseState.GetArgSliceByRef(0); return sizeof(int) + t.MetadataSize + setValue.Length; case RespCommand.SET: @@ -217,8 +217,8 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) return sizeof(int) + t.Length + sizeof(long); case RespCommand.SETRANGE: - var offset = input.parseState.GetInt(input.parseStateFirstArgIdx); - var newValue = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx + 1).ReadOnlySpan; + var offset = input.parseState.GetInt(0); + var newValue = input.parseState.GetArgSliceByRef(1).ReadOnlySpan; if (newValue.Length + offset > t.LengthWithoutMetadata) return sizeof(int) + newValue.Length + offset + t.MetadataSize; @@ -232,7 +232,7 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) return sizeof(int) + t.LengthWithoutMetadata + (input.arg1 > 0 ? sizeof(long) : 0); case RespCommand.APPEND: - var valueLength = input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).Length; + var valueLength = input.parseState.GetArgSliceByRef(0).Length; return sizeof(int) + t.Length + valueLength; default: @@ -252,7 +252,7 @@ public int GetRMWModifiedValueLength(ref SpanByte t, ref RawStringInput input) } } - return sizeof(int) + input.parseState.GetArgSliceByRef(input.parseStateFirstArgIdx).Length + + return sizeof(int) + input.parseState.GetArgSliceByRef(0).Length + (input.arg1 == 0 ? 0 : sizeof(long)); } diff --git a/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs b/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs index 9186d19b35..8a28bc1e1e 100644 --- a/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs @@ -108,7 +108,7 @@ bool InPlaceUpdaterWorker(ref byte[] key, ref ObjectInput input, ref IGarnetObje { case GarnetObjectType.Expire: case GarnetObjectType.PExpire: - var expiryValue = input.parseState.GetLong(input.parseStateFirstArgIdx); + var expiryValue = input.parseState.GetLong(0); var optionType = (ExpireOption)input.arg1; var expireAt = input.arg2 == 1; @@ -198,7 +198,7 @@ public bool PostCopyUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetOb { case GarnetObjectType.Expire: case GarnetObjectType.PExpire: - var expiryValue = input.parseState.GetLong(input.parseStateFirstArgIdx); + var expiryValue = input.parseState.GetLong(0); var optionType = (ExpireOption)input.arg1; var expireAt = input.arg2 == 1; diff --git a/libs/server/Storage/Session/MainStore/HyperLogLogOps.cs b/libs/server/Storage/Session/MainStore/HyperLogLogOps.cs index 0791e49295..f09d34e7c9 100644 --- a/libs/server/Storage/Session/MainStore/HyperLogLogOps.cs +++ b/libs/server/Storage/Session/MainStore/HyperLogLogOps.cs @@ -23,7 +23,7 @@ public unsafe GarnetStatus HyperLogLogAdd(ArgSlice key, string[] eleme parseState.Initialize(1); - var input = new RawStringInput(RespCommand.PFADD, ref parseState, 0, -1, 1); + var input = new RawStringInput(RespCommand.PFADD, ref parseState); var output = stackalloc byte[1]; byte pfaddUpdated = 0; @@ -94,7 +94,7 @@ public unsafe GarnetStatus HyperLogLogLength(ref RawStringInput input, error = false; count = default; - if (input.parseState.Count - input.parseStateFirstArgIdx == 0) + if (input.parseState.Count == 0) return GarnetStatus.OK; var createTransaction = false; @@ -103,12 +103,11 @@ public unsafe GarnetStatus HyperLogLogLength(ref RawStringInput input, { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; - var currTokenIdx = input.parseStateFirstArgIdx; - var dstKey = input.parseState.GetArgSliceByRef(currTokenIdx++); + var dstKey = input.parseState.GetArgSliceByRef(0); txnManager.SaveKeyEntryToLock(dstKey, false, LockType.Exclusive); - while (currTokenIdx < input.parseState.Count) + for (var i = 1; i < input.parseState.Count; i++) { - var currSrcKey = input.parseState.GetArgSliceByRef(currTokenIdx++); + var currSrcKey = input.parseState.GetArgSliceByRef(i); txnManager.SaveKeyEntryToLock(currSrcKey, false, LockType.Shared); } txnManager.Run(true); @@ -128,13 +127,11 @@ public unsafe GarnetStatus HyperLogLogLength(ref RawStringInput input, var srcMergeBuffer = new SpanByteAndMemory(dstReadBuffer, hllBufferSize); var isFirst = false; - var currTokenIdx = input.parseStateFirstArgIdx; - - while (currTokenIdx < input.parseState.Count) + for (var i = 0; i < input.parseState.Count; i++) { var currInput = new RawStringInput(RespCommand.PFCOUNT); - var srcKey = input.parseState.GetArgSliceByRef(currTokenIdx++).SpanByte; + var srcKey = input.parseState.GetArgSliceByRef(i).SpanByte; var status = GET(ref srcKey, ref currInput, ref srcMergeBuffer, ref currLockableContext); // Handle case merging source key does not exist @@ -156,7 +153,7 @@ public unsafe GarnetStatus HyperLogLogLength(ref RawStringInput input, if (!isFirst) { isFirst = true; - if (currTokenIdx == input.parseState.Count) + if (i == input.parseState.Count - 1) count = HyperLogLog.DefaultHLL.Count(srcMergeBuffer.SpanByte.ToPointer()); else Buffer.MemoryCopy(srcHLL, dstHLL, sbSrcHLL.Length, sbSrcHLL.Length); @@ -165,7 +162,7 @@ public unsafe GarnetStatus HyperLogLogLength(ref RawStringInput input, HyperLogLog.DefaultHLL.TryMerge(srcHLL, dstHLL, sbDstHLL.Length); - if (currTokenIdx == input.parseState.Count) + if (i == input.parseState.Count - 1) { count = HyperLogLog.DefaultHLL.Count(dstHLL); } @@ -190,7 +187,7 @@ public unsafe GarnetStatus HyperLogLogMerge(ref RawStringInput input, out bool e { error = false; - if (input.parseState.Count - input.parseStateFirstArgIdx == 0) + if (input.parseState.Count == 0) return GarnetStatus.OK; var createTransaction = false; @@ -199,12 +196,11 @@ public unsafe GarnetStatus HyperLogLogMerge(ref RawStringInput input, out bool e { Debug.Assert(txnManager.state == TxnState.None); createTransaction = true; - var currTokenIdx = input.parseStateFirstArgIdx; - var dstKey = input.parseState.GetArgSliceByRef(currTokenIdx++); + var dstKey = input.parseState.GetArgSliceByRef(0); txnManager.SaveKeyEntryToLock(dstKey, false, LockType.Exclusive); - while (currTokenIdx < input.parseState.Count) + for (var i = 1; i < input.parseState.Count; i++) { - var currSrcKey = input.parseState.GetArgSliceByRef(currTokenIdx++); + var currSrcKey = input.parseState.GetArgSliceByRef(i); txnManager.SaveKeyEntryToLock(currSrcKey, false, LockType.Shared); } txnManager.Run(true); @@ -217,17 +213,16 @@ public unsafe GarnetStatus HyperLogLogMerge(ref RawStringInput input, out bool e sectorAlignedMemoryHll1 ??= new SectorAlignedMemory(hllBufferSize + sectorAlignedMemoryPoolAlignment, sectorAlignedMemoryPoolAlignment); var readBuffer = sectorAlignedMemoryHll1.GetValidPointer(); - var currTokenIdx = input.parseStateFirstArgIdx; - var dstKey = input.parseState.GetArgSliceByRef(currTokenIdx++).SpanByte; + var dstKey = input.parseState.GetArgSliceByRef(0).SpanByte; - while (currTokenIdx < input.parseState.Count) + for (var i = 1; i < input.parseState.Count; i++) { #region readSrcHLL var currInput = new RawStringInput(RespCommand.PFMERGE); var mergeBuffer = new SpanByteAndMemory(readBuffer, hllBufferSize); - var srcKey = input.parseState.GetArgSliceByRef(currTokenIdx++).SpanByte; + var srcKey = input.parseState.GetArgSliceByRef(i).SpanByte; var status = GET(ref srcKey, ref currInput, ref mergeBuffer, ref currLockableContext); // Handle case merging source key does not exist @@ -239,6 +234,7 @@ public unsafe GarnetStatus HyperLogLogMerge(ref RawStringInput input, out bool e error = true; break; } + #endregion #region mergeToDst diff --git a/libs/server/Storage/Session/MainStore/MainStoreOps.cs b/libs/server/Storage/Session/MainStore/MainStoreOps.cs index fceaf24d88..f4f333963c 100644 --- a/libs/server/Storage/Session/MainStore/MainStoreOps.cs +++ b/libs/server/Storage/Session/MainStore/MainStoreOps.cs @@ -846,8 +846,8 @@ public unsafe GarnetStatus EXPIRE(ArgSlice key, ref Ra var expiryAt = respCommand == RespCommand.PEXPIREAT || respCommand == RespCommand.EXPIREAT; var header = new RespInputHeader(type); - var objInput = new ObjectInput(header, ref input.parseState, input.parseStateFirstArgIdx, - input.parseStateLastArgIdx, (int)input.arg1, expiryAt ? 1 : 0); + + var objInput = new ObjectInput(header, ref input.parseState, arg1: (int)input.arg1, arg2: expiryAt ? 1 : 0); // Retry on object store var objOutput = new GarnetObjectStoreOutput { spanByteAndMemory = output }; @@ -946,7 +946,7 @@ public unsafe GarnetStatus EXPIRE(ArgSlice key, long e // Build parse state parseState.InitializeWithArgument(expirySlice); - var input = new RawStringInput(respCommand, ref parseState, 0, -1, (byte)expireOption); + var input = new RawStringInput(respCommand, ref parseState, arg1: (byte)expireOption); var _key = key.SpanByte; var status = context.RMW(ref _key, ref input, ref output); @@ -969,7 +969,7 @@ public unsafe GarnetStatus EXPIRE(ArgSlice key, long e var expiryAt = respCommand == RespCommand.PEXPIREAT || respCommand == RespCommand.EXPIREAT; var header = new RespInputHeader(type); - var objInput = new ObjectInput(header, ref parseState, 0, -1, (byte)expireOption, expiryAt ? 1 : 0); + var objInput = new ObjectInput(header, ref parseState, arg1: (byte)expireOption, arg2: expiryAt ? 1 : 0); // Retry on object store var objOutput = new GarnetObjectStoreOutput { spanByteAndMemory = output }; diff --git a/libs/server/Storage/Session/ObjectStore/Common.cs b/libs/server/Storage/Session/ObjectStore/Common.cs index b2cb99ff7c..ab00fa79f8 100644 --- a/libs/server/Storage/Session/ObjectStore/Common.cs +++ b/libs/server/Storage/Session/ObjectStore/Common.cs @@ -183,7 +183,7 @@ public unsafe GarnetStatus ObjectScan(GarnetObjectType objectTyp // Prepare the input var header = new RespInputHeader(objectType); - var input = new ObjectInput(header, ref parseState, 0, -1, (int)cursor, ObjectScanCountLimit); + var input = new ObjectInput(header, ref parseState, arg1: (int)cursor, arg2: ObjectScanCountLimit); switch (objectType) { diff --git a/libs/server/Storage/Session/ObjectStore/SetOps.cs b/libs/server/Storage/Session/ObjectStore/SetOps.cs index fc3a998185..4d0a47382e 100644 --- a/libs/server/Storage/Session/ObjectStore/SetOps.cs +++ b/libs/server/Storage/Session/ObjectStore/SetOps.cs @@ -727,7 +727,6 @@ public unsafe GarnetStatus SetIsMember(ArgSlice key, ArgSlice[] if (key.Length == 0) return GarnetStatus.OK; - var parseState = new SessionParseState(); parseState.InitializeWithArguments(members); // Prepare the input @@ -735,7 +734,7 @@ public unsafe GarnetStatus SetIsMember(ArgSlice key, ArgSlice[] { type = GarnetObjectType.Set, SetOp = SetOperation.SMISMEMBER, - }, ref parseState, 0); + }, ref parseState); var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(null) }; var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectContext, ref outputFooter); diff --git a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs index a378f62001..b29697cc31 100644 --- a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs +++ b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs @@ -515,7 +515,7 @@ public unsafe GarnetStatus SortedSetRange(ArgSlice key, ArgSlice // Prepare the input var header = new RespInputHeader(GarnetObjectType.SortedSet) { SortedSetOp = sortedOperation }; var inputArg = 2; // Default RESP server protocol version - var input = new ObjectInput(header, ref parseState, 0, -1, inputArg); + var input = new ObjectInput(header, ref parseState, arg1: inputArg); var outputFooter = new GarnetObjectStoreOutput { spanByteAndMemory = new SpanByteAndMemory(null) }; var status = ReadObjectStoreOperationWithOutput(key.ToArray(), ref input, ref objectContext, ref outputFooter); diff --git a/test/Garnet.test/DeleteTxn.cs b/test/Garnet.test/DeleteTxn.cs index f2cc2f9de2..3c31b965fe 100644 --- a/test/Garnet.test/DeleteTxn.cs +++ b/test/Garnet.test/DeleteTxn.cs @@ -19,14 +19,14 @@ sealed class DeleteTxn : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { var offset = 0; - AddKey(GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset), LockType.Exclusive, false); + AddKey(GetNextArg(ref procInput.parseState, ref offset), LockType.Exclusive, false); return true; } public override void Main(TGarnetApi api, ref CustomProcedureInput procInput, ref MemoryResult output) { var offset = 0; - var key = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + var key = GetNextArg(ref procInput.parseState, ref offset); api.DELETE(key, StoreType.Main); WriteSimpleString(ref output, "SUCCESS"); } diff --git a/test/Garnet.test/ObjectExpiryTxn.cs b/test/Garnet.test/ObjectExpiryTxn.cs index 2bf22bcfeb..090321f096 100644 --- a/test/Garnet.test/ObjectExpiryTxn.cs +++ b/test/Garnet.test/ObjectExpiryTxn.cs @@ -19,15 +19,15 @@ sealed class ObjectExpiryTxn : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { var offset = 0; - AddKey(GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset), LockType.Exclusive, true); + AddKey(GetNextArg(ref procInput.parseState, ref offset), LockType.Exclusive, true); return true; } public override void Main(TGarnetApi api, ref CustomProcedureInput procInput, ref MemoryResult output) { var offset = 0; - var key = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); - var expiryMs = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + var key = GetNextArg(ref procInput.parseState, ref offset); + var expiryMs = GetNextArg(ref procInput.parseState, ref offset); api.EXPIRE(key, expiryMs, out _, StoreType.Object); WriteSimpleString(ref output, "SUCCESS"); diff --git a/test/Garnet.test/RespAdminCommandsTests.cs b/test/Garnet.test/RespAdminCommandsTests.cs index 19963ba9b3..76b64289f7 100644 --- a/test/Garnet.test/RespAdminCommandsTests.cs +++ b/test/Garnet.test/RespAdminCommandsTests.cs @@ -523,6 +523,16 @@ public void SeEchoWithMessageTest() ClassicAssert.AreEqual(expectedResponse, actualValue); } + [Test] + public void SeEchoWithMultiTokenMessageTest() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var expectedResponse = "\"HELLO WORLD!\""; + var actualValue = db.Execute("ECHO", "\"HELLO WORLD!\"").ToString(); + ClassicAssert.AreEqual(expectedResponse, actualValue); + } + [Test] public void SeTimeCommandTest() diff --git a/test/Garnet.test/SortedSetRemoveTxn.cs b/test/Garnet.test/SortedSetRemoveTxn.cs index d4821ea41a..1a2d341dd6 100644 --- a/test/Garnet.test/SortedSetRemoveTxn.cs +++ b/test/Garnet.test/SortedSetRemoveTxn.cs @@ -19,7 +19,7 @@ sealed class SortedSetRemoveTxn : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { var offset = 0; - var subscriptionContainerKey = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + var subscriptionContainerKey = GetNextArg(ref procInput.parseState, ref offset); AddKey(subscriptionContainerKey, LockType.Exclusive, true); return true; @@ -28,8 +28,8 @@ public override bool Prepare(TGarnetReadApi api, ref CustomProce public override void Main(TGarnetApi api, ref CustomProcedureInput procInput, ref MemoryResult output) { var offset = 0; - var subscriptionContainerKey = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); - var subscriptionContainerEntry = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + var subscriptionContainerKey = GetNextArg(ref procInput.parseState, ref offset); + var subscriptionContainerEntry = GetNextArg(ref procInput.parseState, ref offset); api.SortedSetRemove(subscriptionContainerKey, subscriptionContainerEntry, out _); diff --git a/test/Garnet.test/TestProcedureBitmap.cs b/test/Garnet.test/TestProcedureBitmap.cs index b9b3e3d65e..9e12a7cd35 100644 --- a/test/Garnet.test/TestProcedureBitmap.cs +++ b/test/Garnet.test/TestProcedureBitmap.cs @@ -22,11 +22,11 @@ sealed class TestProcedureBitmap : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { var offset = 0; - var bitmapA = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); - GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); - GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); - var destinationKey = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); - var bitmapB = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + var bitmapA = GetNextArg(ref procInput.parseState, ref offset); + GetNextArg(ref procInput.parseState, ref offset); + GetNextArg(ref procInput.parseState, ref offset); + var destinationKey = GetNextArg(ref procInput.parseState, ref offset); + var bitmapB = GetNextArg(ref procInput.parseState, ref offset); if (bitmapA.Length == 0) return false; @@ -49,11 +49,11 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p BitmapOperation[] bitwiseOps = [BitmapOperation.AND, BitmapOperation.OR, BitmapOperation.XOR]; //get paramaters - var bitmapA = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); - var offsetArgument = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); - var bitValueArgument = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); - var destinationKeyBitOp = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); - var bitmapB = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + var bitmapA = GetNextArg(ref procInput.parseState, ref offset); + var offsetArgument = GetNextArg(ref procInput.parseState, ref offset); + var bitValueArgument = GetNextArg(ref procInput.parseState, ref offset); + var destinationKeyBitOp = GetNextArg(ref procInput.parseState, ref offset); + var bitmapB = GetNextArg(ref procInput.parseState, ref offset); //simple set and get for bitmaps api.StringSetBit(bitmapA, offsetArgument, bitValueArgument.ToArray()[0] == '1', out _); diff --git a/test/Garnet.test/TestProcedureHLL.cs b/test/Garnet.test/TestProcedureHLL.cs index 99696ba188..a47f4b74ff 100644 --- a/test/Garnet.test/TestProcedureHLL.cs +++ b/test/Garnet.test/TestProcedureHLL.cs @@ -21,7 +21,7 @@ sealed class TestProcedureHLL : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { var offset = 0; - var hll = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + var hll = GetNextArg(ref procInput.parseState, ref offset); if (hll.Length == 0) return false; @@ -36,7 +36,7 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p var elements = new string[7]; var result = true; - var hll = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + var hll = GetNextArg(ref procInput.parseState, ref offset); if (hll.Length == 0) result = false; @@ -45,7 +45,7 @@ public override void Main(TGarnetApi api, ref CustomProcedureInput p { for (var i = 0; i < elements.Length; i++) { - elements[i] = Encoding.ASCII.GetString(GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset).ToArray()); + elements[i] = Encoding.ASCII.GetString(GetNextArg(ref procInput.parseState, ref offset).ToArray()); } api.HyperLogLogAdd(hll, elements, out var resultPfAdd); result = resultPfAdd; diff --git a/test/Garnet.test/TestProcedureHash.cs b/test/Garnet.test/TestProcedureHash.cs index 9d7b29a8e4..570a5f3312 100644 --- a/test/Garnet.test/TestProcedureHash.cs +++ b/test/Garnet.test/TestProcedureHash.cs @@ -22,7 +22,7 @@ sealed class TestProcedureHash : CustomTransactionProcedure public override bool Prepare(TGarnetReadApi api, ref CustomProcedureInput procInput) { var offset = 0; - var setA = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + var setA = GetNextArg(ref procInput.parseState, ref offset); if (setA.Length == 0) return false; @@ -43,15 +43,15 @@ private static bool TestAPI(TGarnetApi api, ref CustomProcedureInput var pairs = new (ArgSlice field, ArgSlice value)[6]; var fields = new ArgSlice[pairs.Length]; - var myHash = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + var myHash = GetNextArg(ref procInput.parseState, ref offset); if (myHash.Length == 0) return false; for (var i = 0; i < pairs.Length; i++) { - pairs[i].field = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); - pairs[i].value = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + pairs[i].field = GetNextArg(ref procInput.parseState, ref offset); + pairs[i].value = GetNextArg(ref procInput.parseState, ref offset); fields[i] = pairs[i].field; } @@ -111,7 +111,7 @@ private static bool TestAPI(TGarnetApi api, ref CustomProcedureInput return false; // HDEL - var elementRemove = GetNextArg(ref procInput.parseState, procInput.parseStateFirstArgIdx, ref offset); + var elementRemove = GetNextArg(ref procInput.parseState, ref offset); status = api.HashDelete(myHash, elementRemove, out count); if (status != GarnetStatus.OK || count != 1) return false; From bda4f3cf9a32d22408089703c8b10fbde1b37e7e Mon Sep 17 00:00:00 2001 From: Roman Kuzmin Date: Mon, 28 Oct 2024 18:26:36 +0000 Subject: [PATCH 13/15] Fix HINCRBYFLOAT precision, use double, not float (#753) --- libs/server/Objects/Hash/HashObjectImpl.cs | 4 ++-- test/Garnet.test/RespHashTests.cs | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/libs/server/Objects/Hash/HashObjectImpl.cs b/libs/server/Objects/Hash/HashObjectImpl.cs index 937df4c85e..674aebfd08 100644 --- a/libs/server/Objects/Hash/HashObjectImpl.cs +++ b/libs/server/Objects/Hash/HashObjectImpl.cs @@ -393,7 +393,7 @@ private void HashIncrement(ref ObjectInput input, ref SpanByteAndMemory output) } else { - if (!NumUtils.TryParse(incrSlice.ReadOnlySpan, out float incr)) + if (!NumUtils.TryParse(incrSlice.ReadOnlySpan, out double incr)) { while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_NOT_VALID_FLOAT, ref curr, end)) ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, @@ -405,7 +405,7 @@ private void HashIncrement(ref ObjectInput input, ref SpanByteAndMemory output) if (valueExists) { - if (!NumUtils.TryParse(value, out float result)) + if (!NumUtils.TryParse(value, out double result)) { while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_HASH_VALUE_IS_NOT_FLOAT, ref curr, end)) diff --git a/test/Garnet.test/RespHashTests.cs b/test/Garnet.test/RespHashTests.cs index 992e33fc15..e4a7b9b8d2 100644 --- a/test/Garnet.test/RespHashTests.cs +++ b/test/Garnet.test/RespHashTests.cs @@ -273,6 +273,16 @@ public void CanDoHashDecrement() ClassicAssert.AreEqual(-3, result); } + [Test] + public void CheckHashIncrementDoublePrecision() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + db.HashSet("user:user1", [new HashEntry("Field1", "1.1111111111")]); + var result = db.HashIncrement(new RedisKey("user:user1"), new RedisValue("Field1"), 2.2222222222); + ClassicAssert.AreEqual(3.3333333333, result, 1e-15); + } + [Test] public void CanDoHSETNXCommand() { From 4e3b462830d3b7f7f706938b108c127c7b6a7a7d Mon Sep 17 00:00:00 2001 From: Vijay Nirmal Date: Tue, 29 Oct 2024 00:26:23 +0530 Subject: [PATCH 14/15] [Compatibility] Added ZDIFFSTORE command (#732) * Added ZDIFFSTORE command * Tied to fix ClusterSlotVerificationTests * Review comments * Review comment fix * Review command fix --------- Co-authored-by: Tal Zaccai --- libs/resources/RespCommandsDocs.json | 30 ++++ libs/resources/RespCommandsInfo.json | 38 +++++ libs/server/API/GarnetApiObjectCommands.cs | 5 + libs/server/API/IGarnetApi.cs | 9 ++ libs/server/Resp/Objects/SortedSetCommands.cs | 53 ++++++- libs/server/Resp/Parser/RespCommand.cs | 5 + libs/server/Resp/RespServerSession.cs | 1 + .../Session/ObjectStore/SortedSetOps.cs | 131 ++++++++++++++---- .../CommandInfoUpdater/SupportedCommand.cs | 1 + .../RedirectTests/BaseCommand.cs | 31 +++++ .../ClusterSlotVerificationTests.cs | 7 + test/Garnet.test/Resp/ACL/RespCommandTests.cs | 15 ++ test/Garnet.test/RespSortedSetTests.cs | 76 ++++++++++ website/docs/commands/api-compatibility.md | 2 +- website/docs/commands/data-structures.md | 18 +++ 15 files changed, 390 insertions(+), 32 deletions(-) diff --git a/libs/resources/RespCommandsDocs.json b/libs/resources/RespCommandsDocs.json index 2315aff5b0..8be78e9a83 100644 --- a/libs/resources/RespCommandsDocs.json +++ b/libs/resources/RespCommandsDocs.json @@ -5310,6 +5310,36 @@ } ] }, + { + "Command": "ZDIFFSTORE", + "Name": "ZDIFFSTORE", + "Summary": "Stores the difference of multiple sorted sets in a key.", + "Group": "SortedSet", + "Complexity": "O(L \u002B (N-K)log(N)) worst case where L is the total number of elements in all the sets, N is the size of the first set, and K is the size of the result set.", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "DESTINATION", + "DisplayText": "destination", + "Type": "Key", + "KeySpecIndex": 0 + }, + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "NUMKEYS", + "DisplayText": "numkeys", + "Type": "Integer" + }, + { + "TypeDiscriminator": "RespCommandKeyArgument", + "Name": "KEY", + "DisplayText": "key", + "Type": "Key", + "ArgumentFlags": "Multiple", + "KeySpecIndex": 1 + } + ] + }, { "Command": "ZINCRBY", "Name": "ZINCRBY", diff --git a/libs/resources/RespCommandsInfo.json b/libs/resources/RespCommandsInfo.json index 9ff0a95864..978ca9be1b 100644 --- a/libs/resources/RespCommandsInfo.json +++ b/libs/resources/RespCommandsInfo.json @@ -4056,6 +4056,44 @@ } ] }, + { + "Command": "ZDIFFSTORE", + "Name": "ZDIFFSTORE", + "Arity": -4, + "Flags": "DenyOom, MovableKeys, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "SortedSet, Slow, Write", + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Flags": "OW, Update" + }, + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 2 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysKeyNum", + "KeyNumIdx": 0, + "FirstKey": 1, + "KeyStep": 1 + }, + "Flags": "RO, Access" + } + ] + }, { "Command": "ZINCRBY", "Name": "ZINCRBY", diff --git a/libs/server/API/GarnetApiObjectCommands.cs b/libs/server/API/GarnetApiObjectCommands.cs index e045363cb3..dd67c7c9cc 100644 --- a/libs/server/API/GarnetApiObjectCommands.cs +++ b/libs/server/API/GarnetApiObjectCommands.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; using System.Collections.Generic; using Tsavorite.core; @@ -129,6 +130,10 @@ public GarnetStatus SortedSetRange(ArgSlice key, ArgSlice min, ArgSlice max, Sor public GarnetStatus SortedSetDifference(ArgSlice[] keys, out Dictionary pairs) => storageSession.SortedSetDifference(keys, out pairs); + /// + public GarnetStatus SortedSetDifferenceStore(ArgSlice destinationKey, ReadOnlySpan keys, out int count) + => storageSession.SortedSetDifferenceStore(destinationKey, keys, out count); + /// public GarnetStatus SortedSetScan(ArgSlice key, long cursor, string match, int count, out ArgSlice[] items) => storageSession.ObjectScan(GarnetObjectType.SortedSet, key, cursor, match, count, out items, ref objectContext); diff --git a/libs/server/API/IGarnetApi.cs b/libs/server/API/IGarnetApi.cs index e271407c63..7cb2b98480 100644 --- a/libs/server/API/IGarnetApi.cs +++ b/libs/server/API/IGarnetApi.cs @@ -473,6 +473,15 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi /// GarnetStatus SortedSetRemoveRangeByRank(ArgSlice key, int start, int stop, out int countRemoved); + /// + /// Computes the difference between the first and all successive sorted sets and store resulting pairs in the output key. + /// + /// + /// + /// + /// + GarnetStatus SortedSetDifferenceStore(ArgSlice destinationKey, ReadOnlySpan keys, out int count); + /// /// Adds geospatial items (longitude, latitude, name) to the specified key. /// diff --git a/libs/server/Resp/Objects/SortedSetCommands.cs b/libs/server/Resp/Objects/SortedSetCommands.cs index 93d7b81b40..9a5a9053b4 100644 --- a/libs/server/Resp/Objects/SortedSetCommands.cs +++ b/libs/server/Resp/Objects/SortedSetCommands.cs @@ -744,7 +744,7 @@ private unsafe bool SortedSetDifference(ref TGarnetApi storageApi) return AbortWithWrongNumberOfArguments("ZDIFF"); } - //number of keys + // Number of keys if (!parseState.TryGetInt(0, out var nKeys)) { while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER, ref dcurr, dend)) @@ -827,5 +827,56 @@ private unsafe bool SortedSetDifference(ref TGarnetApi storageApi) return true; } + + /// + /// Computes a difference operation between the first and all successive sorted sets and store + /// and returns the result to the client. + /// The total number of input keys is specified. + /// + /// + /// + /// + private unsafe bool SortedSetDifferenceStore(ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count < 3) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.ZDIFFSTORE)); + } + + // Number of keys + if (!parseState.TryGetInt(1, out var nKeys)) + { + while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER, ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (parseState.Count - 2 != nKeys) + { + while (!RespWriteUtils.WriteError(CmdStrings.RESP_SYNTAX_ERROR, ref dcurr, dend)) + SendAndReset(); + return true; + } + + var destination = parseState.GetArgSliceByRef(0); + var keys = parseState.Parameters.Slice(2, nKeys); + + var status = storageApi.SortedSetDifferenceStore(destination, keys, out var count); + + switch (status) + { + case GarnetStatus.WRONGTYPE: + while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_WRONG_TYPE, ref dcurr, dend)) + SendAndReset(); + break; + default: + while (!RespWriteUtils.WriteInteger(count, ref dcurr, dend)) + SendAndReset(); + break; + } + + return true; + } } } \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index 7cd956ebb3..2036d29a0a 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -154,6 +154,7 @@ public enum RespCommand : ushort SUNIONSTORE, UNLINK, ZADD, + ZDIFFSTORE, ZINCRBY, ZPOPMAX, ZPOPMIN, @@ -1355,6 +1356,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.SMISMEMBER; } + else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nZDIF"u8) && *(uint*)(ptr + 9) == MemoryMarshal.Read("FSTORE\r\n"u8)) + { + return RespCommand.ZDIFFSTORE; + } break; case 11: diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index f0b44d0026..c97f1b9c9f 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -602,6 +602,7 @@ private bool ProcessArrayCommands(RespCommand cmd, ref TGarnetApi st RespCommand.ZPOPMIN => SortedSetPop(cmd, ref storageApi), RespCommand.ZRANDMEMBER => SortedSetRandomMember(ref storageApi), RespCommand.ZDIFF => SortedSetDifference(ref storageApi), + RespCommand.ZDIFFSTORE => SortedSetDifferenceStore(ref storageApi), RespCommand.ZREVRANGE => SortedSetRange(cmd, ref storageApi), RespCommand.ZREVRANGEBYSCORE => SortedSetRange(cmd, ref storageApi), RespCommand.ZSCAN => ObjectScan(GarnetObjectType.SortedSet, ref storageApi), diff --git a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs index b29697cc31..eb517ad668 100644 --- a/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs +++ b/libs/server/Storage/Session/ObjectStore/SortedSetOps.cs @@ -539,7 +539,7 @@ public unsafe GarnetStatus SortedSetRange(ArgSlice key, ArgSlice /// /// /// - public unsafe GarnetStatus SortedSetDifference(ArgSlice[] keys, out Dictionary pairs) + public unsafe GarnetStatus SortedSetDifference(ReadOnlySpan keys, out Dictionary pairs) { pairs = default; @@ -561,47 +561,76 @@ public unsafe GarnetStatus SortedSetDifference(ArgSlice[] keys, out Dictionary + /// Computes the difference between the first and all successive sorted sets and store resulting pairs in the destination key. + /// + /// + /// + /// + /// + public GarnetStatus SortedSetDifferenceStore(ArgSlice destinationKey, ReadOnlySpan keys, out int count) + { + count = default; + + if (keys.Length == 0) + return GarnetStatus.OK; + + var createTransaction = false; + + if (txnManager.state != TxnState.Running) + { + Debug.Assert(txnManager.state == TxnState.None); + createTransaction = true; + txnManager.SaveKeyEntryToLock(destinationKey, true, LockType.Exclusive); + foreach (var item in keys) + txnManager.SaveKeyEntryToLock(item, true, LockType.Shared); + _ = txnManager.Run(true); + } + + var objectContext = txnManager.ObjectStoreLockableContext; + + try + { + var status = SortedSetDifference(keys, ref objectContext, out var pairs); + + if (status != GarnetStatus.OK) { - if (firstObj.garnetObject is not SortedSetObject firstSortedSet) - { - return GarnetStatus.WRONGTYPE; - } + return GarnetStatus.WRONGTYPE; + } - if (keys.Length == 1) - { - pairs = firstSortedSet.Dictionary; - return GarnetStatus.OK; - } + count = pairs?.Count ?? 0; - // read the rest of the keys - for (var item = 1; item < keys.Length; item++) + if (count > 0) + { + SortedSetObject newSetObject = new(); + foreach (var (element, score) in pairs) { - statusOp = GET(keys[item].ToArray(), out var nextObj, ref objectContext); - if (statusOp != GarnetStatus.OK) - continue; - - if (nextObj.garnetObject is not SortedSetObject nextSortedSet) - { - pairs = default; - return GarnetStatus.WRONGTYPE; - } - - if (pairs == default) - pairs = SortedSetObject.CopyDiff(firstSortedSet.Dictionary, nextSortedSet.Dictionary); - else - SortedSetObject.InPlaceDiff(pairs, nextSortedSet.Dictionary); + newSetObject.Add(element, score); } + _ = SET(destinationKey.ToArray(), newSetObject, ref objectContext); + } + else + { + _ = EXPIRE(destinationKey, TimeSpan.Zero, out _, StoreType.Object, ExpireOption.None, + ref lockableContext, ref objectContext); } + + return status; } finally { if (createTransaction) txnManager.Commit(true); } - - return GarnetStatus.OK; } /// @@ -856,5 +885,47 @@ public GarnetStatus SortedSetRandomMember(byte[] key, ref Object public GarnetStatus SortedSetScan(byte[] key, ref ObjectInput input, ref GarnetObjectStoreOutput outputFooter, ref TObjectContext objectStoreContext) where TObjectContext : ITsavoriteContext => ReadObjectStoreOperationWithOutput(key, ref input, ref objectStoreContext, ref outputFooter); + + private GarnetStatus SortedSetDifference(ReadOnlySpan keys, ref TObjectContext objectContext, out Dictionary pairs) + where TObjectContext : ITsavoriteContext + { + pairs = default; + + var statusOp = GET(keys[0].ToArray(), out var firstObj, ref objectContext); + if (statusOp == GarnetStatus.OK) + { + if (firstObj.garnetObject is not SortedSetObject firstSortedSet) + { + return GarnetStatus.WRONGTYPE; + } + + if (keys.Length == 1) + { + pairs = firstSortedSet.Dictionary; + return GarnetStatus.OK; + } + + // read the rest of the keys + for (var item = 1; item < keys.Length; item++) + { + statusOp = GET(keys[item].ToArray(), out var nextObj, ref objectContext); + if (statusOp != GarnetStatus.OK) + continue; + + if (nextObj.garnetObject is not SortedSetObject nextSortedSet) + { + pairs = default; + return GarnetStatus.WRONGTYPE; + } + + if (pairs == default) + pairs = SortedSetObject.CopyDiff(firstSortedSet.Dictionary, nextSortedSet.Dictionary); + else + SortedSetObject.InPlaceDiff(pairs, nextSortedSet.Dictionary); + } + } + + return GarnetStatus.OK; + } } } \ No newline at end of file diff --git a/playground/CommandInfoUpdater/SupportedCommand.cs b/playground/CommandInfoUpdater/SupportedCommand.cs index 5ccf20af5e..33a3d82b23 100644 --- a/playground/CommandInfoUpdater/SupportedCommand.cs +++ b/playground/CommandInfoUpdater/SupportedCommand.cs @@ -254,6 +254,7 @@ public class SupportedCommand new("ZCARD", RespCommand.ZCARD), new("ZCOUNT", RespCommand.ZCOUNT), new("ZDIFF", RespCommand.ZDIFF), + new("ZDIFFSTORE", RespCommand.ZDIFFSTORE), new("ZINCRBY", RespCommand.ZINCRBY), new("ZLEXCOUNT", RespCommand.ZLEXCOUNT), new("ZMSCORE", RespCommand.ZMSCORE), diff --git a/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs b/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs index dcd12f1f6a..d1fc00d071 100644 --- a/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs +++ b/test/Garnet.test.cluster/RedirectTests/BaseCommand.cs @@ -1864,6 +1864,37 @@ public override string[] GetSingleSlotRequest() public override ArraySegment[] SetupSingleSlotRequest() => throw new NotImplementedException(); } + + internal class ZDIFFSTORE : BaseCommand + { + public override bool IsArrayCommand => true; + public override bool ArrayResponse => false; + public override string Command => nameof(ZDIFFSTORE); + + public override string[] GetSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + // ZDIFFSTORE c 2 a b + return [ssk[0], "2", ssk[1], ssk[2]]; + } + + public override string[] GetCrossSlotRequest() + { + var csk = GetCrossSlotKeys; + return [csk[0], "2", csk[1], csk[2]]; + } + + public override ArraySegment[] SetupSingleSlotRequest() + { + var ssk = GetSingleSlotKeys; + var setup = new ArraySegment[3]; + setup[0] = new ArraySegment(["ZADD", ssk[1], "1", "a"]); + setup[1] = new ArraySegment(["ZADD", ssk[2], "2", "b"]); + setup[2] = new ArraySegment(["ZADD", ssk[3], "3", "c"]); + return setup; + } + } + #endregion #region HashCommands diff --git a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs b/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs index 429eb6543e..bbd539ed3d 100644 --- a/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs +++ b/test/Garnet.test.cluster/RedirectTests/ClusterSlotVerificationTests.cs @@ -104,6 +104,7 @@ public class ClusterSlotVerificationTests new ZREMRANGEBYRANK(), new ZRANDMEMBER(), new ZDIFF(), + new ZDIFFSTORE(), new HSET(), new HGET(), new HGETALL(), @@ -281,6 +282,7 @@ public virtual void OneTimeTearDown() [TestCase("ZREMRANGEBYRANK")] [TestCase("ZRANDMEMBER")] [TestCase("ZDIFF")] + [TestCase("ZDIFFSTORE")] [TestCase("HSET")] [TestCase("HGET")] [TestCase("HGETALL")] @@ -418,6 +420,7 @@ void GarnetClientSessionClusterDown(BaseCommand command) [TestCase("ZREMRANGEBYRANK")] [TestCase("ZRANDMEMBER")] [TestCase("ZDIFF")] + [TestCase("ZDIFFSTORE")] [TestCase("HSET")] [TestCase("HGET")] [TestCase("HGETALL")] @@ -565,6 +568,7 @@ void GarnetClientSessionOK(BaseCommand command) [TestCase("ZREMRANGEBYRANK")] [TestCase("ZRANDMEMBER")] [TestCase("ZDIFF")] + [TestCase("ZDIFFSTORE")] [TestCase("HSET")] [TestCase("HGET")] [TestCase("HGETALL")] @@ -704,6 +708,7 @@ void GarnetClientSessionCrossslotTest(BaseCommand command) [TestCase("ZREMRANGEBYRANK")] [TestCase("ZRANDMEMBER")] [TestCase("ZDIFF")] + [TestCase("ZDIFFSTORE")] [TestCase("HSET")] [TestCase("HGET")] [TestCase("HGETALL")] @@ -850,6 +855,7 @@ void GarnetClientSessionMOVEDTest(BaseCommand command) [TestCase("ZREMRANGEBYRANK")] [TestCase("ZRANDMEMBER")] [TestCase("ZDIFF")] + [TestCase("ZDIFFSTORE")] [TestCase("HSET")] [TestCase("HGET")] [TestCase("HGETALL")] @@ -1013,6 +1019,7 @@ void GarnetClientSessionASKTest(BaseCommand command) [TestCase("ZREMRANGEBYRANK")] [TestCase("ZRANDMEMBER")] [TestCase("ZDIFF")] + [TestCase("ZDIFFSTORE")] [TestCase("HSET")] [TestCase("HGET")] [TestCase("HGETALL")] diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/Garnet.test/Resp/ACL/RespCommandTests.cs index c71932497f..45679ef550 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/Garnet.test/Resp/ACL/RespCommandTests.cs @@ -5877,6 +5877,21 @@ static async Task DoZDiffMultiAsync(GarnetClient client) } } + [Test] + public async Task ZDiffStoreACLsAsync() + { + await CheckCommandsAsync( + "ZDIFFSTORE", + [DoZDiffStoreAsync] + ); + + static async Task DoZDiffStoreAsync(GarnetClient client) + { + var val = await client.ExecuteForLongResultAsync("ZDIFFSTORE", ["keyZ", "2", "foo", "bar"]); + ClassicAssert.AreEqual(0, val); + } + } + [Test] public async Task ZScanACLsAsync() { diff --git a/test/Garnet.test/RespSortedSetTests.cs b/test/Garnet.test/RespSortedSetTests.cs index 69deb4ea32..52889b6f66 100644 --- a/test/Garnet.test/RespSortedSetTests.cs +++ b/test/Garnet.test/RespSortedSetTests.cs @@ -940,6 +940,7 @@ public void CheckSortedSetOperationsOnWrongTypeObjectSE() var db = redis.GetDatabase(0); var keys = new[] { new RedisKey("user1:obj1"), new RedisKey("user1:obj2") }; + var destinationKey = new RedisKey("user1:objA"); var key1Values = new[] { new RedisValue("Hello"), new RedisValue("World") }; var key2Values = new[] { new RedisValue("Hola"), new RedisValue("Mundo") }; var values = new[] { key1Values, key2Values }; @@ -989,6 +990,8 @@ public void CheckSortedSetOperationsOnWrongTypeObjectSE() RespTestsUtils.CheckCommandOnWrongTypeObjectSE(() => db.SortedSetRandomMember(keys[1])); // ZDIFF RespTestsUtils.CheckCommandOnWrongTypeObjectSE(() => db.SortedSetCombine(SetOperation.Difference, keys)); + // ZDIFFSTORE + RespTestsUtils.CheckCommandOnWrongTypeObjectSE(() => db.SortedSetCombineAndStore(SetOperation.Difference, destinationKey, keys)); // ZSCAN RespTestsUtils.CheckCommandOnWrongTypeObjectSE(() => db.SortedSetScan(keys[1], new RedisValue("*")).FirstOrDefault()); //ZMSCORE @@ -1032,6 +1035,57 @@ public void CanDoZDiff() ClassicAssert.AreEqual(0, diffWithScore.Length); } + [Test] + [TestCase(false)] + [TestCase(true)] + public void CheckSortedSetDifferenceStoreSE(bool isDestinationKeyExisting) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var keys = new[] { new RedisKey("user1:obj1"), new RedisKey("user1:obj2") }; + var destinationKey = new RedisKey("user1:objA"); + var key1Values = new[] { new SortedSetEntry("Hello", 1), new SortedSetEntry("World", 2) }; + var key2Values = new[] { new SortedSetEntry("Hello", 5), new SortedSetEntry("Mundo", 7) }; + var expectedValue = new SortedSetEntry("World", 2); + + // Set up sorted sets + db.SortedSetAdd(keys[0], key1Values); + db.SortedSetAdd(keys[1], key2Values); + + if (isDestinationKeyExisting) + { + db.SortedSetAdd(destinationKey, key1Values); // Set up destination key to overwrite if exists + } + + var actualCount = db.SortedSetCombineAndStore(SetOperation.Difference, destinationKey, keys); + ClassicAssert.AreEqual(1, actualCount); + + var actualMembers = db.SortedSetRangeByScoreWithScores(destinationKey); + ClassicAssert.AreEqual(1, actualMembers.Length); + ClassicAssert.AreEqual(expectedValue.Element.ToString(), actualMembers[0].Element.ToString()); + ClassicAssert.AreEqual(expectedValue.Score, actualMembers[0].Score); + } + + [Test] + public void CheckSortedSetDifferenceStoreWithNoMatchSE() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var keys = new[] { new RedisKey("user1:obj1"), new RedisKey("user1:obj2") }; + var destinationKey = new RedisKey("user1:objA"); + + // Set up sorted sets + db.SortedSetAdd(destinationKey, "Dummy", 10); // Set up destination key to overwrite if exists + + var actualCount = db.SortedSetCombineAndStore(SetOperation.Difference, destinationKey, keys); + ClassicAssert.AreEqual(0, actualCount); + + var actualMembers = db.SortedSetRangeByScoreWithScores(destinationKey); + ClassicAssert.AreEqual(0, actualMembers.Length); + } + #endregion #region LightClientTests @@ -2027,6 +2081,28 @@ public void CanUseZDiff(int bytesSent) actualValue = Encoding.ASCII.GetString(zdiffResult).Substring(0, expectedResponse.Length); } + [Test] + [TestCase(10)] + [TestCase(30)] + [TestCase(100)] + public void CanUseZDiffSTORE(int bytesSent) + { + using var lightClientRequest = TestUtils.CreateRequest(); + + //zdiff withscores + var zdiffResult = lightClientRequest.SendCommandChunks("ZDIFFSTORE desKey 2 dadi seconddadi", bytesSent); + var expectedResponse = ":0\r\n"; + var actualValue = Encoding.ASCII.GetString(zdiffResult).Substring(0, expectedResponse.Length); + ClassicAssert.AreEqual(expectedResponse, actualValue); + + lightClientRequest.SendCommand("ZADD dadi 1 uno 2 due 3 tre 4 quattro 5 cinque 6 sei"); + lightClientRequest.SendCommand("ZADD seconddadi 1 uno 2 due 3 tre 4 quattro"); + + zdiffResult = lightClientRequest.SendCommandChunks("ZDIFFSTORE desKey 2 dadi seconddadi", bytesSent); + expectedResponse = "1\r\n"; + actualValue = Encoding.ASCII.GetString(zdiffResult).Substring(0, expectedResponse.Length); + } + [Test] [TestCase(10)] [TestCase(30)] diff --git a/website/docs/commands/api-compatibility.md b/website/docs/commands/api-compatibility.md index fa6b2bb6d8..1e398e0525 100644 --- a/website/docs/commands/api-compatibility.md +++ b/website/docs/commands/api-compatibility.md @@ -325,7 +325,7 @@ Note that this list is subject to change as we continue to expand our API comman | | [ZCARD](data-structures.md#zcard) | ➕ | | | | [ZCOUNT](data-structures.md#zcount) | ➕ | | | | [ZDIFF](data-structures.md#zdiff) | ➕ | | -| | ZDIFFSTORE | ➖ | | +| | [ZDIFFSTORE](data-structures.md#zdiffstore) | ➕ | | | | [ZINCRBY](data-structures.md#zincrby) | ➕ | | | | ZINTER | ➖ | | | | ZINTERCARD | ➖ | | diff --git a/website/docs/commands/data-structures.md b/website/docs/commands/data-structures.md index 7c5a4d4a4c..bf9a634fd8 100644 --- a/website/docs/commands/data-structures.md +++ b/website/docs/commands/data-structures.md @@ -760,6 +760,24 @@ Keys that do not exist are considered to be empty sets. --- +### ZDIFFSTORE + +#### Syntax + +```bash + ZDIFFSTORE destination numkeys key [key ...] +``` + +Computes the difference between the first and all successive input sorted sets and stores the result in destination. The total number of input keys is specified by numkeys. + +Keys that do not exist are considered to be empty sets. + +#### Resp Reply + +Integer reply: the number of members in the resulting sorted set at destination. + +--- + ### ZINCRBY #### Syntax From 6d489c6d7570d853c1ef90add86f1958a42e6fa9 Mon Sep 17 00:00:00 2001 From: Vijay Nirmal Date: Tue, 29 Oct 2024 01:27:03 +0530 Subject: [PATCH 15/15] [Compatibility] Added PUBSUB CHANNELS, NUMPAT and NUMSUB commands (#719) * Added PUBSUB CHANNELS, NUMPAT and NUMSUB commands * Code style fix * Review comment fix * Fixed build issue * Review command fix * Review command fix --------- Co-authored-by: Tal Zaccai --- libs/resources/RespCommandsDocs.json | 48 ++++++ libs/resources/RespCommandsInfo.json | 29 ++++ libs/server/PubSub/SubscribeBroker.cs | 158 ++++++++++++++++++ libs/server/Resp/CmdStrings.cs | 5 + libs/server/Resp/Parser/RespCommand.cs | 34 ++++ libs/server/Resp/PubSubCommands.cs | 75 +++++++++ libs/server/Resp/RespServerSession.cs | 3 + .../CommandInfoUpdater/SupportedCommand.cs | 6 + test/Garnet.test/Resp/ACL/RespCommandTests.cs | 47 +++++- test/Garnet.test/RespPubSubTests.cs | 147 +++++++++++++--- website/docs/commands/analytics.md | 46 +++++ website/docs/commands/api-compatibility.md | 6 +- 12 files changed, 580 insertions(+), 24 deletions(-) diff --git a/libs/resources/RespCommandsDocs.json b/libs/resources/RespCommandsDocs.json index 8be78e9a83..426502ee71 100644 --- a/libs/resources/RespCommandsDocs.json +++ b/libs/resources/RespCommandsDocs.json @@ -3858,6 +3858,54 @@ } ] }, + { + "Command": "PUBSUB", + "Name": "PUBSUB", + "Summary": "A container for Pub/Sub commands.", + "Group": "PubSub", + "Complexity": "Depends on subcommand.", + "SubCommands": [ + { + "Command": "PUBSUB_NUMPAT", + "Name": "PUBSUB|NUMPAT", + "Summary": "Returns a count of unique pattern subscriptions.", + "Group": "PubSub", + "Complexity": "O(1)" + }, + { + "Command": "PUBSUB_CHANNELS", + "Name": "PUBSUB|CHANNELS", + "Summary": "Returns the active channels.", + "Group": "PubSub", + "Complexity": "O(N) where N is the number of active channels, and assuming constant time pattern matching (relatively short channels and patterns)", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "PATTERN", + "DisplayText": "pattern", + "Type": "Pattern", + "ArgumentFlags": "Optional" + } + ] + }, + { + "Command": "PUBSUB_NUMSUB", + "Name": "PUBSUB|NUMSUB", + "Summary": "Returns a count of subscribers to channels.", + "Group": "PubSub", + "Complexity": "O(N) for the NUMSUB subcommand, where N is the number of requested channels", + "Arguments": [ + { + "TypeDiscriminator": "RespCommandBasicArgument", + "Name": "CHANNEL", + "DisplayText": "channel", + "Type": "String", + "ArgumentFlags": "Optional, Multiple" + } + ] + } + ] + }, { "Command": "PUNSUBSCRIBE", "Name": "PUNSUBSCRIBE", diff --git a/libs/resources/RespCommandsInfo.json b/libs/resources/RespCommandsInfo.json index 978ca9be1b..2e7732a60e 100644 --- a/libs/resources/RespCommandsInfo.json +++ b/libs/resources/RespCommandsInfo.json @@ -2805,6 +2805,35 @@ "Flags": "Fast, Loading, PubSub, Stale", "AclCategories": "Fast, PubSub" }, + { + "Command": "PUBSUB", + "Name": "PUBSUB", + "Arity": -2, + "AclCategories": "Slow", + "SubCommands": [ + { + "Command": "PUBSUB_CHANNELS", + "Name": "PUBSUB|CHANNELS", + "Arity": -2, + "Flags": "Loading, PubSub, Stale", + "AclCategories": "PubSub, Slow" + }, + { + "Command": "PUBSUB_NUMPAT", + "Name": "PUBSUB|NUMPAT", + "Arity": 2, + "Flags": "Loading, PubSub, Stale", + "AclCategories": "PubSub, Slow" + }, + { + "Command": "PUBSUB_NUMSUB", + "Name": "PUBSUB|NUMSUB", + "Arity": -2, + "Flags": "Loading, PubSub, Stale", + "AclCategories": "PubSub, Slow" + } + ] + }, { "Command": "PUNSUBSCRIBE", "Name": "PUNSUBSCRIBE", diff --git a/libs/server/PubSub/SubscribeBroker.cs b/libs/server/PubSub/SubscribeBroker.cs index a3985a5eb5..53d073b922 100644 --- a/libs/server/PubSub/SubscribeBroker.cs +++ b/libs/server/PubSub/SubscribeBroker.cs @@ -2,8 +2,10 @@ // Licensed under the MIT license. using System; +using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Threading; using System.Threading.Tasks; using Garnet.common; @@ -408,6 +410,162 @@ public unsafe void Publish(byte* key, byte* value, int valueLength, bool ascii = log.Enqueue(logEntryBytes); } + /// + /// Retrieves the collection of channels that have active subscriptions. + /// + /// The collection of channels. + public unsafe void Channels(ref ObjectInput input, ref SpanByteAndMemory output) + { + var isMemory = false; + MemoryHandle ptrHandle = default; + var ptr = output.SpanByte.ToPointer(); + + var curr = ptr; + var end = curr + output.Length; + + try + { + if (subscriptions is null || subscriptions.Count == 0) + { + while (!RespWriteUtils.WriteEmptyArray(ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + return; + } + + if (input.parseState.Count == 0) + { + while (!RespWriteUtils.WriteArrayLength(subscriptions.Count, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + foreach (var key in subscriptions.Keys) + { + while (!RespWriteUtils.WriteBulkString(key.AsSpan().Slice(sizeof(int)), ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + } + return; + } + + // Below WriteArrayLength is primarily to move the start of the buffer to the max length that is required to write the array length. The actual length is written in the below line. + // This is done to avoid multiple two passes over the subscriptions or new array allocation if we use single pass over the subscriptions + var totalArrayHeaderLen = 0; + while (!RespWriteUtils.WriteArrayLength(subscriptions.Count, ref curr, end, out var _, out totalArrayHeaderLen)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + var noOfFoundChannels = 0; + var pattern = input.parseState.GetArgSliceByRef(0).SpanByte; + var patternPtr = pattern.ToPointer() - sizeof(int); + *(int*)patternPtr = pattern.Length; + + foreach (var key in subscriptions.Keys) + { + fixed (byte* keyPtr = key) + { + var endKeyPtr = keyPtr; + var _patternPtr = patternPtr; + if (keySerializer.Match(ref keySerializer.ReadKeyByRef(ref endKeyPtr), true, ref keySerializer.ReadKeyByRef(ref _patternPtr), true)) + { + while (!RespWriteUtils.WriteSimpleString(key.AsSpan().Slice(sizeof(int)), ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + noOfFoundChannels++; + } + } + } + + if (noOfFoundChannels == 0) + { + curr = ptr; + while (!RespWriteUtils.WriteEmptyArray(ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + return; + } + + // Below code is to write the actual array length in the buffer + // And move the array elements to the start of the new array length if new array length is less than the max array length that we orginally write in the above line + var newTotalArrayHeaderLen = 0; + var _ptr = ptr; + // ReallocateOutput is not needed here as there should be always be available space in the output buffer as we have already written the max array length + _ = RespWriteUtils.WriteArrayLength(noOfFoundChannels, ref _ptr, end, out var _, out newTotalArrayHeaderLen); + + Debug.Assert(totalArrayHeaderLen >= newTotalArrayHeaderLen, "newTotalArrayHeaderLen can't be bigger than totalArrayHeaderLen as we have already written max array lenght in the buffer"); + if (totalArrayHeaderLen != newTotalArrayHeaderLen) + { + var remainingLength = (curr - ptr) - totalArrayHeaderLen; + Buffer.MemoryCopy(ptr + totalArrayHeaderLen, ptr + newTotalArrayHeaderLen, remainingLength, remainingLength); + curr = curr - (totalArrayHeaderLen - newTotalArrayHeaderLen); + } + } + finally + { + if (isMemory) + ptrHandle.Dispose(); + output.Length = (int)(curr - ptr); + } + } + + /// + /// Retrieves the number of pattern subscriptions. + /// + /// The number of pattern subscriptions. + public int NumPatternSubscriptions() + { + return prefixSubscriptions?.Count ?? 0; + } + + /// + /// PUBSUB NUMSUB + /// + /// + /// + public unsafe void NumSubscriptions(ref ObjectInput input, ref SpanByteAndMemory output) + { + var isMemory = false; + MemoryHandle ptrHandle = default; + var ptr = output.SpanByte.ToPointer(); + + var curr = ptr; + var end = curr + output.Length; + + try + { + var numOfChannels = input.parseState.Count; + if (subscriptions is null || numOfChannels == 0) + { + while (!RespWriteUtils.WriteEmptyArray(ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + return; + } + + while (!RespWriteUtils.WriteArrayLength(numOfChannels * 2, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + var currChannelIdx = 0; + while (currChannelIdx < numOfChannels) + { + var channelArg = input.parseState.GetArgSliceByRef(currChannelIdx); + var channelSpan = channelArg.SpanByte; + var channelPtr = channelSpan.ToPointer() - sizeof(int); // Memory would have been already pinned + *(int*)channelPtr = channelSpan.Length; + + while (!RespWriteUtils.WriteBulkString(channelArg.ReadOnlySpan, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + var channel = new Span(channelPtr, channelSpan.Length + sizeof(int)).ToArray(); + + subscriptions.TryGetValue(channel, out var subscriptionDict); + while (!RespWriteUtils.WriteInteger(subscriptionDict is null ? 0 : subscriptionDict.Count, ref curr, end)) + ObjectUtils.ReallocateOutput(ref output, ref isMemory, ref ptr, ref ptrHandle, ref curr, ref end); + + currChannelIdx++; + } + } + finally + { + if (isMemory) + ptrHandle.Dispose(); + output.Length = (int)(curr - ptr); + } + } + /// public void Dispose() { diff --git a/libs/server/Resp/CmdStrings.cs b/libs/server/Resp/CmdStrings.cs index 4815c1c8e9..4dc7121f11 100644 --- a/libs/server/Resp/CmdStrings.cs +++ b/libs/server/Resp/CmdStrings.cs @@ -104,6 +104,10 @@ static partial class CmdStrings public static ReadOnlySpan rank => "rank"u8; public static ReadOnlySpan MAXLEN => "MAXLEN"u8; public static ReadOnlySpan maxlen => "maxlen"u8; + public static ReadOnlySpan PUBSUB => "PUBSUB"u8; + public static ReadOnlySpan CHANNELS => "CHANNELS"u8; + public static ReadOnlySpan NUMPAT => "NUMPAT"u8; + public static ReadOnlySpan NUMSUB => "NUMSUB"u8; /// /// Response strings @@ -207,6 +211,7 @@ static partial class CmdStrings public const string GenericParamShouldBeGreaterThanZero = "ERR {0} should be greater than 0"; public const string GenericUnknownClientType = "ERR Unknown client type '{0}'"; public const string GenericErrDuplicateFilter = "ERR Filter '{0}' defined multiple times"; + public const string GenericPubSubCommandDisabled = "ERR {0} is disabled, enable it with --pubsub option."; /// /// Response errors while scripting diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index 2036d29a0a..e9c912ff5f 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -179,6 +179,11 @@ public enum RespCommand : ushort PING, + // Pub/Sub commands + PUBSUB, + PUBSUB_CHANNELS, + PUBSUB_NUMPAT, + PUBSUB_NUMSUB, PUBLISH, SUBSCRIBE, PSUBSCRIBE, @@ -1967,6 +1972,35 @@ private RespCommand SlowParseCommand(ref int count, ref ReadOnlySpan speci return RespCommand.MODULE_LOADCS; } } + else if (command.SequenceEqual(CmdStrings.PUBSUB)) + { + Span subCommand = GetCommand(out var gotSubCommand); + if (!gotSubCommand) + { + success = false; + return RespCommand.NONE; + } + + count--; + AsciiUtils.ToUpperInPlace(subCommand); + if (subCommand.SequenceEqual(CmdStrings.CHANNELS)) + { + return RespCommand.PUBSUB_CHANNELS; + } + else if (subCommand.SequenceEqual(CmdStrings.NUMSUB)) + { + return RespCommand.PUBSUB_NUMSUB; + } + else if (subCommand.SequenceEqual(CmdStrings.NUMPAT)) + { + return RespCommand.PUBSUB_NUMPAT; + } + else + { + success = false; + return RespCommand.NONE; + } + } else { // Custom commands should have never been set when we reach this point diff --git a/libs/server/Resp/PubSubCommands.cs b/libs/server/Resp/PubSubCommands.cs index 2bd16051f5..0c1c666d12 100644 --- a/libs/server/Resp/PubSubCommands.cs +++ b/libs/server/Resp/PubSubCommands.cs @@ -370,5 +370,80 @@ private bool NetworkPUNSUBSCRIBE() } return true; } + + private bool NetworkPUBSUB_CHANNELS() + { + if (parseState.Count > 1) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.PUBSUB_CHANNELS)); + } + + if (subscribeBroker is null) + { + while (!RespWriteUtils.WriteError(string.Format(CmdStrings.GenericPubSubCommandDisabled, "PUBSUB CHANNELS"), ref dcurr, dend)) + SendAndReset(); + return true; + } + + var input = new ObjectInput() + { + parseState = parseState + }; + var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + subscribeBroker.Channels(ref input, ref output); + + if (!output.IsSpanByte) + SendAndReset(output.Memory, output.Length); + else + dcurr += output.Length; + + return true; + } + + private bool NetworkPUBSUB_NUMPAT() + { + if (parseState.Count > 0) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.PUBSUB_NUMPAT)); + } + + if (subscribeBroker is null) + { + while (!RespWriteUtils.WriteError(string.Format(CmdStrings.GenericPubSubCommandDisabled, "PUBSUB NUMPAT"), ref dcurr, dend)) + SendAndReset(); + return true; + } + + var numPatSubs = subscribeBroker.NumPatternSubscriptions(); + + while (!RespWriteUtils.WriteInteger(numPatSubs, ref dcurr, dend)) + SendAndReset(); + + return true; + } + + private bool NetworkPUBSUB_NUMSUB() + { + if (subscribeBroker is null) + { + while (!RespWriteUtils.WriteError(string.Format(CmdStrings.GenericPubSubCommandDisabled, "PUBSUB NUMSUB"), ref dcurr, dend)) + SendAndReset(); + return true; + } + + var input = new ObjectInput + { + parseState = parseState + }; + var output = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + subscribeBroker.NumSubscriptions(ref input, ref output); + + if (!output.IsSpanByte) + SendAndReset(output.Memory, output.Length); + else + dcurr += output.Length; + + return true; + } } } \ No newline at end of file diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index c97f1b9c9f..89e921c51d 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -580,6 +580,9 @@ private bool ProcessArrayCommands(RespCommand cmd, ref TGarnetApi st RespCommand.PSUBSCRIBE => NetworkPSUBSCRIBE(), RespCommand.UNSUBSCRIBE => NetworkUNSUBSCRIBE(), RespCommand.PUNSUBSCRIBE => NetworkPUNSUBSCRIBE(), + RespCommand.PUBSUB_CHANNELS => NetworkPUBSUB_CHANNELS(), + RespCommand.PUBSUB_NUMSUB => NetworkPUBSUB_NUMSUB(), + RespCommand.PUBSUB_NUMPAT => NetworkPUBSUB_NUMPAT(), // Custom Object Commands RespCommand.COSCAN => ObjectScan(GarnetObjectType.All, ref storageApi), // Sorted Set commands diff --git a/playground/CommandInfoUpdater/SupportedCommand.cs b/playground/CommandInfoUpdater/SupportedCommand.cs index 33a3d82b23..2aaf0ad74b 100644 --- a/playground/CommandInfoUpdater/SupportedCommand.cs +++ b/playground/CommandInfoUpdater/SupportedCommand.cs @@ -199,6 +199,12 @@ public class SupportedCommand new("PSUBSCRIBE", RespCommand.PSUBSCRIBE), new("PTTL", RespCommand.PTTL), new("PUBLISH", RespCommand.PUBLISH), + new("PUBSUB", RespCommand.PUBSUB, + [ + new("PUBSUB|CHANNELS", RespCommand.PUBSUB_CHANNELS), + new("PUBSUB|NUMPAT", RespCommand.PUBSUB_NUMPAT), + new("PUBSUB|NUMSUB", RespCommand.PUBSUB_NUMSUB), + ]), new("PUNSUBSCRIBE", RespCommand.PUNSUBSCRIBE), new("REGISTERCS", RespCommand.REGISTERCS), new("QUIT", RespCommand.QUIT), diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/Garnet.test/Resp/ACL/RespCommandTests.cs index 45679ef550..99f4616dc8 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/Garnet.test/Resp/ACL/RespCommandTests.cs @@ -83,7 +83,7 @@ public void AllCommandsCovered() ClassicAssert.IsTrue(RespCommandsInfo.TryGetRespCommandNames(out IReadOnlySet advertisedCommands), "Couldn't get advertised RESP commands"); // TODO: See if these commands could be identified programmatically - IEnumerable withOnlySubCommands = ["ACL", "CLIENT", "CLUSTER", "CONFIG", "LATENCY", "MEMORY", "MODULE"]; + IEnumerable withOnlySubCommands = ["ACL", "CLIENT", "CLUSTER", "CONFIG", "LATENCY", "MEMORY", "MODULE", "PUBSUB"]; IEnumerable notCoveredByACLs = allInfo.Where(static x => x.Value.Flags.HasFlag(RespCommandFlags.NoAuth)).Select(static kv => kv.Key); // Check tests against RespCommandsInfo @@ -4460,6 +4460,51 @@ static async Task DoPublishAsync(GarnetClient client) } } + [Test] + public async Task PubSubChannelsACLsAsync() + { + await CheckCommandsAsync( + "PUBSUB CHANNELS", + [DoPubSubChannelsAsync] + ); + + static async Task DoPubSubChannelsAsync(GarnetClient client) + { + var count = await client.ExecuteForStringArrayResultAsync("PUBSUB", ["CHANNELS"]); + CollectionAssert.IsEmpty(count); + } + } + + [Test] + public async Task PubSubNumPatACLsAsync() + { + await CheckCommandsAsync( + "PUBSUB NUMPAT", + [DoPubSubNumPatAsync] + ); + + static async Task DoPubSubNumPatAsync(GarnetClient client) + { + var count = await client.ExecuteForLongResultAsync("PUBSUB", ["NUMPAT"]); + ClassicAssert.AreEqual(0, count); + } + } + + [Test] + public async Task PubSubNumSubACLsAsync() + { + await CheckCommandsAsync( + "PUBSUB NUMSUB", + [DoPubSubNumSubAsync] + ); + + static async Task DoPubSubNumSubAsync(GarnetClient client) + { + var count = await client.ExecuteForStringArrayResultAsync("PUBSUB", ["NUMSUB"]); + CollectionAssert.IsEmpty(count); + } + } + [Test] public async Task ReadOnlyACLsAsync() { diff --git a/test/Garnet.test/RespPubSubTests.cs b/test/Garnet.test/RespPubSubTests.cs index d54cabd527..43d851a72a 100644 --- a/test/Garnet.test/RespPubSubTests.cs +++ b/test/Garnet.test/RespPubSubTests.cs @@ -2,7 +2,9 @@ // Licensed under the MIT license. using System; +using System.Linq; using System.Threading; +using System.Threading.Channels; using NUnit.Framework; using NUnit.Framework.Legacy; using StackExchange.Redis; @@ -36,28 +38,18 @@ public void BasicSUBSCRIBE() using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); var sub = subRedis.GetSubscriber(); var db = redis.GetDatabase(0); + string value = "published message"; ManualResetEvent evt = new(false); - sub.Subscribe(RedisChannel.Pattern("messages"), (channel, message) => + SubscribeAndPublish(sub, db, RedisChannel.Literal("messages"), RedisChannel.Literal("messages"), value, onSubscribe: (channel, message) => { ClassicAssert.AreEqual("messages", (string)channel); - ClassicAssert.AreEqual("published message", (string)message); + ClassicAssert.AreEqual(value, (string)message); evt.Set(); }); - // Repeat to work-around bug in StackExchange.Redis subscribe behavior - // where it returns before the SUBSCRIBE call is processed. - int repeat = 5; - while (true) - { - db.Publish(RedisChannel.Pattern("messages"), "published message"); - var ret = evt.WaitOne(TimeSpan.FromSeconds(1)); - if (ret) break; - repeat--; - ClassicAssert.IsTrue(repeat != 0, "Timeout waiting for subsciption receive"); - } - sub.Unsubscribe(RedisChannel.Pattern("messages")); + sub.Unsubscribe(RedisChannel.Literal("messages")); } [Test] @@ -68,15 +60,15 @@ public void BasicPSUBSCRIBE() var sub = subRedis.GetSubscriber(); var db = redis.GetDatabase(0); - string glob = "com.messages.*"; - string actual = "com.messages.testmessage"; + string glob = "messagesA*"; + string actual = "messagesAtest"; string value = "published message"; var channel = new RedisChannel(glob, RedisChannel.PatternMode.Pattern); ManualResetEvent evt = new(false); - sub.Subscribe(channel, (receivedChannel, message) => + SubscribeAndPublish(sub, db, channel, RedisChannel.Pattern(actual), value, (receivedChannel, message) => { ClassicAssert.AreEqual(glob, (string)channel); ClassicAssert.AreEqual(actual, (string)receivedChannel); @@ -84,18 +76,133 @@ public void BasicPSUBSCRIBE() evt.Set(); }); + sub.Unsubscribe(channel); + } + + [Test] + public void BasicPUBSUB_CHANNELS() + { + using var subRedis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var sub = subRedis.GetSubscriber(); + var db = redis.GetDatabase(0); + var server = redis.GetServers()[0]; + + var channelA = "messagesAtest"; + var channelB = "messagesB"; + + SubscribeAndPublish(sub, db, RedisChannel.Literal(channelA), RedisChannel.Pattern(channelA)); + SubscribeAndPublish(sub, db, RedisChannel.Literal(channelB), RedisChannel.Pattern(channelB)); + + var result = server.SubscriptionChannels(); + string[] expectedResult = [channelA, channelB]; + CollectionAssert.IsSubsetOf(expectedResult, result.Select(x => x.ToString())); + + result = server.SubscriptionChannels(RedisChannel.Pattern("messages*")); + expectedResult = [channelA, channelB]; + CollectionAssert.AreEquivalent(expectedResult, result.Select(x => x.ToString())); + + result = server.SubscriptionChannels(RedisChannel.Pattern("messages?test")); + expectedResult = [channelA]; + CollectionAssert.AreEquivalent(expectedResult, result.Select(x => x.ToString())); + + result = server.SubscriptionChannels(RedisChannel.Pattern("messagesC*")); + ClassicAssert.AreEqual(0, result.Length); + + sub.Unsubscribe(RedisChannel.Literal(channelA)); + sub.Unsubscribe(RedisChannel.Literal(channelB)); + } + + [Test] + public void BasicPUBSUB_NUMPAT() + { + using var subRedis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var sub = subRedis.GetSubscriber(); + var db = redis.GetDatabase(0); + var server = redis.GetServers()[0]; + + string glob = "com.messages.*"; + string globB = "com.messagesB.*"; + string actual = "com.messages.testmessage"; + string actualB = "com.messagesB.testmessage"; + string value = "published message"; + + var channel = new RedisChannel(glob, RedisChannel.PatternMode.Pattern); + var channelB = new RedisChannel(globB, RedisChannel.PatternMode.Pattern); + + var result = server.SubscriptionPatternCount(); + ClassicAssert.AreEqual(0, result); + + SubscribeAndPublish(sub, db, channel, RedisChannel.Literal(actual), value); + SubscribeAndPublish(sub, db, channelB, RedisChannel.Literal(actualB), value); + + result = server.SubscriptionPatternCount(); + ClassicAssert.AreEqual(2, result); + + sub.Unsubscribe(channel); + sub.Unsubscribe(channelB); + } + + [Test] + public void BasicPUBSUB_NUMSUB() + { + using var subRedis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var sub = subRedis.GetSubscriber(); + var db = redis.GetDatabase(0); + var server = redis.GetServers()[0]; + + var multiChannelResult = server.Execute("PUBSUB", ["NUMSUB"]); + ClassicAssert.AreEqual(0, multiChannelResult.Length); + + multiChannelResult = server.Execute("PUBSUB", ["NUMSUB", "messagesA", "messagesB"]); + ClassicAssert.AreEqual(4, multiChannelResult.Length); + ClassicAssert.AreEqual("messagesA", multiChannelResult[0].ToString()); + ClassicAssert.AreEqual("0", multiChannelResult[1].ToString()); + ClassicAssert.AreEqual("messagesB", multiChannelResult[2].ToString()); + ClassicAssert.AreEqual("0", multiChannelResult[3].ToString()); + + SubscribeAndPublish(sub, db, RedisChannel.Literal("messagesA")); + SubscribeAndPublish(sub, db, RedisChannel.Literal("messagesB")); + + var result = server.SubscriptionSubscriberCount(RedisChannel.Literal("messagesA")); + ClassicAssert.AreEqual(1, result); + + multiChannelResult = server.Execute("PUBSUB", ["NUMSUB", "messagesA", "messagesB"]); + ClassicAssert.AreEqual(4, multiChannelResult.Length); + ClassicAssert.AreEqual("messagesA", multiChannelResult[0].ToString()); + ClassicAssert.AreEqual("1", multiChannelResult[1].ToString()); + ClassicAssert.AreEqual("messagesB", multiChannelResult[2].ToString()); + ClassicAssert.AreEqual("1", multiChannelResult[3].ToString()); + + sub.Unsubscribe(RedisChannel.Literal("messagesA")); + sub.Unsubscribe(RedisChannel.Literal("messagesB")); + } + + private void SubscribeAndPublish(ISubscriber sub, IDatabase db, RedisChannel channel, RedisChannel? publishChannel = null, string message = null, Action onSubscribe = null) + { + message ??= "published message"; + publishChannel ??= channel; + ManualResetEvent evt = new(false); + sub.Subscribe(channel, (receivedChannel, receivedMessage) => + { + onSubscribe?.Invoke(receivedChannel, receivedMessage); + evt.Set(); + }); + + // Doing publish to make sure the channel is subscribed // Repeat to work-around bug in StackExchange.Redis subscribe behavior // where it returns before the SUBSCRIBE call is processed. int repeat = 5; while (true) { - db.Publish(RedisChannel.Pattern(actual), value); + db.Publish(publishChannel.Value, message); var ret = evt.WaitOne(TimeSpan.FromSeconds(1)); if (ret) break; repeat--; - ClassicAssert.IsTrue(repeat != 0, "Timeout waiting for subsciption receive"); + ClassicAssert.IsTrue(repeat != 0, "Timeout waiting for subscription receive"); } - sub.Unsubscribe(channel); } } } \ No newline at end of file diff --git a/website/docs/commands/analytics.md b/website/docs/commands/analytics.md index 7ab10a7c4d..69a1c43851 100644 --- a/website/docs/commands/analytics.md +++ b/website/docs/commands/analytics.md @@ -218,6 +218,52 @@ Posts a message to the given channel. Integer Reply: the number of clients that received the message. --- + +### PUBSUB CHANNELS +#### Syntax + +```bash +PUBSUB CHANNELS [pattern] +``` + +Lists the currently active channels. An active channel is a Pub/Sub channel with one or more subscribers (excluding clients subscribed to patterns). + +#### Resp Reply + +Array reply: a list of active channels, optionally matching the specified pattern. + +--- + +### PUBSUB NUMPAT +#### Syntax + +```bash +PUBSUB NUMPAT +``` + +Returns the number of unique patterns that are subscribed to by clients (that are performed using the PSUBSCRIBE command). + +#### Resp Reply + +Integer reply: the number of patterns all the clients are subscribed to. + +--- + +### PUBSUB NUMSUB +#### Syntax + +```bash +PUBSUB NUMSUB [channel [channel ...]] +``` + +Returns the number of subscribers (exclusive of clients subscribed to patterns) for the specified channels. + +#### Resp Reply + +Array reply: the number of subscribers per channel, each even element (including the 0th) is channel name, each odd element is the number of subscribers + +--- + ### PUNSUBSCRIBE #### Syntax diff --git a/website/docs/commands/api-compatibility.md b/website/docs/commands/api-compatibility.md index 1e398e0525..e44e169500 100644 --- a/website/docs/commands/api-compatibility.md +++ b/website/docs/commands/api-compatibility.md @@ -248,10 +248,10 @@ Note that this list is subject to change as we continue to expand our API comman | | REFCOUNT | ➖ | | | **PUB/SUB** | [PSUBSCRIBE](analytics.md#psubscribe) | ➕ | | | | [PUBLISH](analytics.md#publish) | ➕ | | -| | PUBSUB CHANNELS | ➖ | | +| | [PUBSUB CHANNELS](analytics.md#pubsub-channels) | ➖ | | | | PUBSUB HELP | ➖ | | -| | PUBSUB NUMPAT | ➖ | | -| | PUBSUB NUMSUB | ➖ | | +| | [PUBSUB NUMPAT](analytics.md#pubsub-numpat) | ➖ | | +| | [PUBSUB NUMSUB](analytics.md#pubsub-numsub) | ➖ | | | | PUBSUB SHARDCHANNELS | ➖ | | | | PUBSUB SHARDNUMSUB | ➖ | | | | [PUNSUBSCRIBE](analytics.md#punsubscribe) | ➕ | |