Skip to content

Array fields support #81

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Apr 22, 2024
Merged

Array fields support #81

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
f031efe
Added Array fields support
BlackGad Apr 18, 2024
ae8675f
Added unit tests
BlackGad Apr 19, 2024
4f7a753
Submodule update to v2.4
BlackGad Apr 19, 2024
fed64e5
Reverted unnecessary changes
BlackGad Apr 19, 2024
5d525a5
Code style, namings changes. Removed extra array data convertion. Cod…
BlackGad Apr 22, 2024
01bb6c0
Revert "Submodule update to v2.4"
BlackGad Apr 22, 2024
9f1f276
Whitespace stuff
roji Apr 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 137 additions & 0 deletions Milvus.Client.Tests/FieldTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,4 +67,141 @@ public void CreateFloatVectorTest()
Assert.Equal(2, field.RowCount);
Assert.Equal(2, field.Data.Count);
}

[Fact]
public void CreateInt8ArrayTest()
{
var field = FieldData.CreateArray(
"vector",
new sbyte[][]
{
[1, 2],
[3, 4],
});

Assert.Equal(MilvusDataType.Array, field.DataType);
Assert.Equal(MilvusDataType.Int8, field.ElementType);
Assert.Equal(2, field.RowCount);
Assert.Equal(2, field.Data.Count);
}

[Fact]
public void CreateInt16ArrayTest()
{
var field = FieldData.CreateArray(
"vector",
new short[][]
{
[1, 2],
[3, 4],
});

Assert.Equal(MilvusDataType.Array, field.DataType);
Assert.Equal(MilvusDataType.Int16, field.ElementType);
Assert.Equal(2, field.RowCount);
Assert.Equal(2, field.Data.Count);
}

[Fact]
public void CreateInt32ArrayTest()
{
var field = FieldData.CreateArray(
"vector",
new int[][]
{
[1, 2],
[3, 4],
});

Assert.Equal(MilvusDataType.Array, field.DataType);
Assert.Equal(MilvusDataType.Int32, field.ElementType);
Assert.Equal(2, field.RowCount);
Assert.Equal(2, field.Data.Count);
}

[Fact]
public void CreateInt64ArrayTest()
{
var field = FieldData.CreateArray(
"vector",
new long[][]
{
[1, 2],
[3, 4],
});

Assert.Equal(MilvusDataType.Array, field.DataType);
Assert.Equal(MilvusDataType.Int64, field.ElementType);
Assert.Equal(2, field.RowCount);
Assert.Equal(2, field.Data.Count);
}

[Fact]
public void CreateBoolArrayTest()
{
var field = FieldData.CreateArray(
"vector",
new bool[][]
{
[true, false],
[false, false],
});

Assert.Equal(MilvusDataType.Array, field.DataType);
Assert.Equal(MilvusDataType.Bool, field.ElementType);
Assert.Equal(2, field.RowCount);
Assert.Equal(2, field.Data.Count);
}

[Fact]
public void CreateFloatArrayTest()
{
var field = FieldData.CreateArray(
"vector",
new float[][]
{
[1, 2],
[3, 4],
});

Assert.Equal(MilvusDataType.Array, field.DataType);
Assert.Equal(MilvusDataType.Float, field.ElementType);
Assert.Equal(2, field.RowCount);
Assert.Equal(2, field.Data.Count);
}

[Fact]
public void CreateDoubleArrayTest()
{
var field = FieldData.CreateArray(
"vector",
new double[][]
{
[1, 2],
[3, 4],
});

Assert.Equal(MilvusDataType.Array, field.DataType);
Assert.Equal(MilvusDataType.Double, field.ElementType);
Assert.Equal(2, field.RowCount);
Assert.Equal(2, field.Data.Count);
}

//TODO: differentiate VarChar and String somehow
[Fact]
public void CreateVarCharArrayTest()
{
var field = FieldData.CreateArray(
"vector",
new string[][]
{
["3d4d387208e04a9abe77be65e2b7c7b3", "a5502ddb557047968a70ff69720d2dd2"],
["4c246789a91f4b15aa3b26799df61457", "00a23e95823b4f14854ceed5f7059953"],
});

Assert.Equal(MilvusDataType.Array, field.DataType);
Assert.Equal(MilvusDataType.VarChar, field.ElementType);
Assert.Equal(2, field.RowCount);
Assert.Equal(2, field.Data.Count);
}
}
1 change: 1 addition & 0 deletions Milvus.Client.sln.DotSettings
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
<s:Boolean x:Key="/Default/UserDictionary/Words/=pymilvus/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=quantizes/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=quantizing/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=sbytes/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=Tanimoto/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=topk/@EntryIndexedValue">True</s:Boolean>
<s:Boolean x:Key="/Default/UserDictionary/Words/=upserted/@EntryIndexedValue">True</s:Boolean>
Expand Down
187 changes: 187 additions & 0 deletions Milvus.Client/ArrayFieldData.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
namespace Milvus.Client;

/// <summary>
/// Binary Field
/// </summary>
public sealed class ArrayFieldData<TElementData> : FieldData<IReadOnlyList<TElementData>>
{
/// <summary>
/// Construct an array field
/// </summary>
/// <param name="fieldName"></param>
/// <param name="data"></param>
/// <param name="isDynamic"></param>
public ArrayFieldData(string fieldName, IEnumerable<IEnumerable<TElementData>> data, bool isDynamic)
: base(fieldName, data.Select(x => x.ToArray()).ToArray(), MilvusDataType.Array, isDynamic)
{
ElementType = EnsureDataType<TElementData>();
}

/// <summary>
/// Array element type
/// </summary>
public MilvusDataType ElementType { get; }

/// <inheritdoc />
internal override Grpc.FieldData ToGrpcFieldData()
{
Check();

Grpc.FieldData fieldData = new()
{
Type = Grpc.DataType.Array,
IsDynamic = IsDynamic
};

if (FieldName is not null)
{
fieldData.FieldName = FieldName;
}

var arrayArray = new ArrayArray
{
ElementType = (DataType) ElementType,
};

fieldData.Scalars = new ScalarField
{
ArrayData = arrayArray
};

foreach (var array in Data)
{
switch (ElementType)
{
case MilvusDataType.Bool:
Grpc.BoolArray boolData = new();
boolData.Data.AddRange(array as IEnumerable<bool>);
arrayArray.Data.Add(new ScalarField {BoolData = boolData});
break;

case MilvusDataType.Int8:
Grpc.IntArray int8Data = new();
var sbytes = array as IEnumerable<sbyte> ?? Enumerable.Empty<sbyte>();
int8Data.Data.AddRange(sbytes.Select(x => (int) x));
arrayArray.Data.Add(new ScalarField {IntData = int8Data});
break;

case MilvusDataType.Int16:
Grpc.IntArray int16Data = new();
var shorts = array as IEnumerable<short> ?? Enumerable.Empty<short>();
int16Data.Data.AddRange(shorts.Select(x=>(int)x));
arrayArray.Data.Add(new ScalarField {IntData = int16Data});
break;

case MilvusDataType.Int32:
Grpc.IntArray int32Data = new();
int32Data.Data.AddRange(array as IEnumerable<int>);
arrayArray.Data.Add(new ScalarField {IntData = int32Data});
break;

case MilvusDataType.Int64:
Grpc.LongArray int64Data = new();
int64Data.Data.AddRange(array as IEnumerable<long>);
arrayArray.Data.Add(new ScalarField {LongData = int64Data});
break;

case MilvusDataType.Float:
Grpc.FloatArray floatData = new();
floatData.Data.AddRange(array as IEnumerable<float>);
arrayArray.Data.Add(new ScalarField {FloatData = floatData});
break;

case MilvusDataType.Double:
Grpc.DoubleArray doubleData = new();
doubleData.Data.AddRange(array as IEnumerable<double>);
arrayArray.Data.Add(new ScalarField {DoubleData = doubleData});
break;

case MilvusDataType.String:
Grpc.StringArray stringData = new();
stringData.Data.AddRange(array as IEnumerable<string>);
arrayArray.Data.Add(new ScalarField {StringData = stringData});
break;

case MilvusDataType.VarChar:
Grpc.StringArray varcharData = new();
varcharData.Data.AddRange(array as IEnumerable<string>);
arrayArray.Data.Add(new ScalarField {StringData = varcharData});
break;

case MilvusDataType.Json:
Grpc.JSONArray jsonData = new();
var enumerable = array as IEnumerable<string> ?? Enumerable.Empty<string>();
jsonData.Data.AddRange(enumerable.Select(ByteString.CopyFromUtf8));
arrayArray.Data.Add(new ScalarField {JsonData = jsonData});
break;
case MilvusDataType.None:
throw new MilvusException($"ElementType Error:{DataType}");
default:
throw new MilvusException($"ElementType Error:{DataType}, not supported");
}
}

return fieldData;

/*
int dataCount = Data.Count;
if (dataCount == 0)
{
throw new MilvusException("The number of vectors must be positive.");
}

int vectorByteLength = Data[0].Length;
int totalByteLength = vectorByteLength;
for (int i = 1; i < dataCount; i++)
{
int rowLength = Data[i].Length;
if (rowLength != vectorByteLength)
{
throw new MilvusException("All vectors must have the same dimensionality.");
}

checked { totalByteLength += rowLength; }
}

byte[] bytes = ArrayPool<byte>.Shared.Rent(totalByteLength);
int pos = 0;
for (int i = 0; i < dataCount; i++)
{
ReadOnlyMemory<byte> row = Data[i];
row.Span.CopyTo(bytes.AsSpan(pos, row.Length));
pos += row.Length;
}
Debug.Assert(pos == totalByteLength);

var result = new Grpc.FieldData
{
FieldName = FieldName,
Type = (Grpc.DataType)DataType,
Vectors = new Grpc.VectorField
{
BinaryVector = ByteString.CopyFrom(bytes.AsSpan(0, totalByteLength)),
Dim = vectorByteLength * 8,
}
};

ArrayPool<byte>.Shared.Return(bytes);
*/
}

internal override object GetValueAsObject(int index)
=> ElementType switch
{
MilvusDataType.Bool => ((IReadOnlyList<IEnumerable<bool>>) Data)[index],
MilvusDataType.Int8 => ((IReadOnlyList<IEnumerable<sbyte>>) Data)[index],
MilvusDataType.Int16 => ((IReadOnlyList<IEnumerable<short>>) Data)[index],
MilvusDataType.Int32 => ((IReadOnlyList<IEnumerable<int>>) Data)[index],
MilvusDataType.Int64 => ((IReadOnlyList<IEnumerable<long>>) Data)[index],
MilvusDataType.Float => ((IReadOnlyList<IEnumerable<float>>) Data)[index],
MilvusDataType.Double => ((IReadOnlyList<IEnumerable<double>>) Data)[index],
MilvusDataType.String => ((IReadOnlyList<IEnumerable<string>>) Data)[index],
MilvusDataType.VarChar => ((IReadOnlyList<IEnumerable<string>>) Data)[index],

MilvusDataType.None => throw new MilvusException($"DataType Error:{DataType}"),
_ => throw new MilvusException($"DataType Error:{DataType}, not supported")
};
}
5 changes: 5 additions & 0 deletions Milvus.Client/Constants.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,11 @@ internal static class Constants
/// </summary>
internal const string FailedReason = "failed_reason";

/// <summary>
/// Key name.
/// </summary>
internal const string MaxCapacity = "max_capacity";

/// <summary>
/// Files.
/// </summary>
Expand Down
Loading