Skip to content

Commit

Permalink
Bugfix/comprehensive index equality (#496)
Browse files Browse the repository at this point in the history 
* fixing issues with index equality

* fixing hash vector corner case
  • Loading branch information
@slorello89
slorello89 authored Oct 28, 2024
1 parent 1956366 commit 52bff7f
Show file tree
Hide file tree
Showing 3 changed files with 345 additions and 7 deletions.
153 changes: 148 additions & 5 deletions src/Redis.OM/Modeling/RedisIndex.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,29 +20,88 @@ public static class RedisIndex
public static bool IndexDefinitionEquals(this RedisIndexInfo redisIndexInfo, Type type)
{
var serialisedDefinition = SerializeIndex(type);
var existingSet = redisIndexInfo.Attributes?.Select(a => (Property: a.Attribute!, a.Type!)).OrderBy(a => a.Property);
var isJson = redisIndexInfo.IndexDefinition?.Identifier == "JSON";

var currentOffset = 0;
if (serialisedDefinition.Length < 5)
{
throw new ArgumentException($"Could not parse the index definition for type: {type.Name}.");
}

if (redisIndexInfo.IndexName != serialisedDefinition[0])
if (redisIndexInfo.IndexDefinition is null)
{
return false;
}

if (redisIndexInfo.IndexDefinition?.Identifier?.Equals(serialisedDefinition[2], StringComparison.OrdinalIgnoreCase) == false)
// these are properties we cannot process because FT.INFO does not respond with them
var unprocessableProperties = new string[] { "EPSILON", "EF_RUNTIME", "PHONETIC", "STOPWORDS" };

foreach (var property in unprocessableProperties)
{
if (serialisedDefinition.Any(x => x.Equals(property)))
{
throw new ArgumentException($"Could not validate index definition that contains {property}");
}
}

if (redisIndexInfo.IndexName != serialisedDefinition[currentOffset])
{
return false;
}

currentOffset += 2; // skip to the index type at index 2

if (redisIndexInfo.IndexDefinition?.Identifier?.Equals(serialisedDefinition[currentOffset], StringComparison.OrdinalIgnoreCase) == false)
{
return false;
}

currentOffset += 2; // skip to prefix count

if (!int.TryParse(serialisedDefinition[currentOffset], out var numPrefixes))
{
throw new ArgumentException("Could not parse index with unknown number of prefixes");
}

currentOffset += 2; // skip to first prefix

if (redisIndexInfo.IndexDefinition?.Prefixes is null || redisIndexInfo.IndexDefinition.Prefixes.Length != numPrefixes || serialisedDefinition.Skip(currentOffset).Take(numPrefixes).SequenceEqual(redisIndexInfo.IndexDefinition.Prefixes))
{
return false;
}

currentOffset += numPrefixes;

if (redisIndexInfo.IndexDefinition?.Filter is not null && !redisIndexInfo.IndexDefinition.Filter.Equals(serialisedDefinition.ElementAt(currentOffset)))
{
return false;
}

if (redisIndexInfo.IndexDefinition?.Filter is not null)
{
currentOffset += 2;
}

if (redisIndexInfo.IndexDefinition?.DefaultLanguage is not null && !redisIndexInfo.IndexDefinition.DefaultLanguage.Equals(serialisedDefinition.ElementAt(currentOffset)))
{
return false;
}

if (redisIndexInfo.IndexDefinition?.Prefixes.FirstOrDefault().Equals(serialisedDefinition[5]) == false)
if (redisIndexInfo.IndexDefinition?.DefaultLanguage is not null)
{
currentOffset += 2;
}

if (redisIndexInfo.IndexDefinition?.LanguageField is not null && !redisIndexInfo.IndexDefinition.LanguageField.Equals(serialisedDefinition.ElementAt(currentOffset)))
{
return false;
}

if (redisIndexInfo.IndexDefinition?.LanguageField is not null)
{
currentOffset += 2;
}

var target = redisIndexInfo.Attributes?.SelectMany(a =>
{
var attr = new List<string>();
Expand All @@ -58,11 +117,81 @@ public static bool IndexDefinitionEquals(this RedisIndexInfo redisIndexInfo, Typ
attr.Add("AS");
}

if (!isJson && a.Type is not null && a.Type == "VECTOR")
{
attr.Add($"{a.Attribute!}.Vector");
attr.Add("AS");
}

attr.Add(a.Attribute!);

if (a.Type != null)
{
attr.Add(a.Type);
if (a.Type == "TAG")
{
attr.Add("SEPARATOR");
attr.Add(a.Separator ?? "|");
}

if (a.Type == "TEXT")
{
if (a.NoStem == true)
{
attr.Add("NOSTEM");
}

if (a.Weight is not null && a.Weight != "1")
{
attr.Add("WEIGHT");
attr.Add(a.Weight);
}
}

if (a.Type == "VECTOR")
{
if (a.Algorithm is null)
{
throw new InvalidOperationException("Encountered Vector field with no algorithm");
}

attr.Add(a.Algorithm);
if (a.VectorType is null)
{
throw new InvalidOperationException("Encountered vector field with no Vector Type");
}

attr.Add(NumVectorArgs(a).ToString());

attr.Add("TYPE");
attr.Add(a.VectorType);

if (a.Dimension is null)
{
throw new InvalidOperationException("Encountered vector field with no dimension");
}

attr.Add("DIM");
attr.Add(a.Dimension);

if (a.DistanceMetric is not null)
{
attr.Add("DISTANCE_METRIC");
attr.Add(a.DistanceMetric);
}

if (a.M is not null)
{
attr.Add("M");
attr.Add(a.M);
}

if (a.EfConstruction is not null)
{
attr.Add("EF_CONSTRUCTION");
attr.Add(a.EfConstruction);
}
}
}

if (a.Sortable == true)
Expand All @@ -73,7 +202,21 @@ public static bool IndexDefinitionEquals(this RedisIndexInfo redisIndexInfo, Typ
return attr.ToArray();
});

return target.SequenceEqual(serialisedDefinition.Skip(7));
return target.SequenceEqual(serialisedDefinition.Skip(currentOffset));
}

/// <summary>
/// calculates the number of arguments that would be required based to reverse engineer the index based off what
/// is in the Info attribute.
/// </summary>
/// <param name="attr">The attribute.</param>
/// <returns>The number of arguments.</returns>
internal static int NumVectorArgs(this RedisIndexInfo.RedisIndexInfoAttribute attr)
{
var numArgs = 6;
numArgs += attr.M is not null ? 2 : 0;
numArgs += attr.EfConstruction is not null ? 2 : 0;
return numArgs;
}

/// <summary>
Expand Down
71 changes: 71 additions & 0 deletions src/Redis.OM/RedisIndexInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using Redis.OM.Modeling;

namespace Redis.OM
{
Expand Down Expand Up @@ -220,6 +221,9 @@ public RedisIndexInfoIndexDefinition(RedisReply redisReply)
case "key_type": Identifier = value.ToString(CultureInfo.InvariantCulture); break;
case "prefixes": Prefixes = value.ToArray().Select(x => x.ToString(CultureInfo.InvariantCulture)).ToArray(); break;
case "default_score": DefaultScore = value.ToString(CultureInfo.InvariantCulture); break;
case "default_language": DefaultLanguage = value.ToString(CultureInfo.InvariantCulture); break;
case "filter": Filter = value.ToString(CultureInfo.InvariantCulture); break;
case "language_field": LanguageField = value.ToString(CultureInfo.InvariantCulture); break;
}
}
}
Expand All @@ -238,6 +242,21 @@ public RedisIndexInfoIndexDefinition(RedisReply redisReply)
/// Gets default_score.
/// </summary>
public string? DefaultScore { get; }

/// <summary>
/// Gets Filter.
/// </summary>
public string? Filter { get; }

/// <summary>
/// Gets language.
/// </summary>
public string? DefaultLanguage { get; }

/// <summary>
/// Gets LanguageField.
/// </summary>
public string? LanguageField { get; }
}

/// <summary>
Expand Down Expand Up @@ -266,9 +285,21 @@ public RedisIndexInfoAttribute(RedisReply redisReply)
case "attribute": Attribute = value; break;
case "type": Type = value; break;
case "SEPARATOR": Separator = value; break;
case "algorithm": Algorithm = value; break;
case "data_type": VectorType = value; break;
case "dim": Dimension = value; break;
case "distance_metric": DistanceMetric = value; break;
case "M": M = value; break;
case "ef_construction": EfConstruction = value; break;
case "WEIGHT": Weight = value; break;
}
}

if (responseArray.Any(x => ((string)x).Equals("NOSTEM", StringComparison.InvariantCultureIgnoreCase)))
{
NoStem = true;
}

if (responseArray.Select(x => x.ToString())
.Any(x => x.Equals("SORTABLE", StringComparison.InvariantCultureIgnoreCase)))
{
Expand Down Expand Up @@ -300,6 +331,46 @@ public RedisIndexInfoAttribute(RedisReply redisReply)
/// Gets SORTABLE.
/// </summary>
public bool? Sortable { get; }

/// <summary>
/// Gets NOSTEM.
/// </summary>
public bool? NoStem { get; }

/// <summary>
/// Gets weight.
/// </summary>
public string? Weight { get; }

/// <summary>
/// Gets Algorithm.
/// </summary>
public string? Algorithm { get; }

/// <summary>
/// Gets the VectorType.
/// </summary>
public string? VectorType { get; }

/// <summary>
/// Gets Dimension.
/// </summary>
public string? Dimension { get; }

/// <summary>
/// Gets DistanceMetric.
/// </summary>
public string? DistanceMetric { get; }

/// <summary>
/// Gets M.
/// </summary>
public string? M { get; }

/// <summary>
/// Gets EF constructor.
/// </summary>
public string? EfConstruction { get; }
}

/// <summary>
Expand Down
Loading

0 comments on commit 52bff7f

Please sign in to comment.