Skip to content

Commit b1bd801

Browse files
BlackGadBlackGad
authored
Array fields support (#81)
Fixes #77
1 parent de2608d commit b1bd801

File tree

8 files changed

+428
-33
lines changed

8 files changed

+428
-33
lines changed

‎Milvus.Client.Tests/FieldTests.cs

Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,4 +67,141 @@ public void CreateFloatVectorTest()
6767
Assert.Equal(2, field.RowCount);
6868
Assert.Equal(2, field.Data.Count);
6969
}
70+
71+
[Fact]
72+
public void CreateInt8ArrayTest()
73+
{
74+
var field = FieldData.CreateArray(
75+
"vector",
76+
new sbyte[][]
77+
{
78+
[1, 2],
79+
[3, 4],
80+
});
81+
82+
Assert.Equal(MilvusDataType.Array, field.DataType);
83+
Assert.Equal(MilvusDataType.Int8, field.ElementType);
84+
Assert.Equal(2, field.RowCount);
85+
Assert.Equal(2, field.Data.Count);
86+
}
87+
88+
[Fact]
89+
public void CreateInt16ArrayTest()
90+
{
91+
var field = FieldData.CreateArray(
92+
"vector",
93+
new short[][]
94+
{
95+
[1, 2],
96+
[3, 4],
97+
});
98+
99+
Assert.Equal(MilvusDataType.Array, field.DataType);
100+
Assert.Equal(MilvusDataType.Int16, field.ElementType);
101+
Assert.Equal(2, field.RowCount);
102+
Assert.Equal(2, field.Data.Count);
103+
}
104+
105+
[Fact]
106+
public void CreateInt32ArrayTest()
107+
{
108+
var field = FieldData.CreateArray(
109+
"vector",
110+
new int[][]
111+
{
112+
[1, 2],
113+
[3, 4],
114+
});
115+
116+
Assert.Equal(MilvusDataType.Array, field.DataType);
117+
Assert.Equal(MilvusDataType.Int32, field.ElementType);
118+
Assert.Equal(2, field.RowCount);
119+
Assert.Equal(2, field.Data.Count);
120+
}
121+
122+
[Fact]
123+
public void CreateInt64ArrayTest()
124+
{
125+
var field = FieldData.CreateArray(
126+
"vector",
127+
new long[][]
128+
{
129+
[1, 2],
130+
[3, 4],
131+
});
132+
133+
Assert.Equal(MilvusDataType.Array, field.DataType);
134+
Assert.Equal(MilvusDataType.Int64, field.ElementType);
135+
Assert.Equal(2, field.RowCount);
136+
Assert.Equal(2, field.Data.Count);
137+
}
138+
139+
[Fact]
140+
public void CreateBoolArrayTest()
141+
{
142+
var field = FieldData.CreateArray(
143+
"vector",
144+
new bool[][]
145+
{
146+
[true, false],
147+
[false, false],
148+
});
149+
150+
Assert.Equal(MilvusDataType.Array, field.DataType);
151+
Assert.Equal(MilvusDataType.Bool, field.ElementType);
152+
Assert.Equal(2, field.RowCount);
153+
Assert.Equal(2, field.Data.Count);
154+
}
155+
156+
[Fact]
157+
public void CreateFloatArrayTest()
158+
{
159+
var field = FieldData.CreateArray(
160+
"vector",
161+
new float[][]
162+
{
163+
[1, 2],
164+
[3, 4],
165+
});
166+
167+
Assert.Equal(MilvusDataType.Array, field.DataType);
168+
Assert.Equal(MilvusDataType.Float, field.ElementType);
169+
Assert.Equal(2, field.RowCount);
170+
Assert.Equal(2, field.Data.Count);
171+
}
172+
173+
[Fact]
174+
public void CreateDoubleArrayTest()
175+
{
176+
var field = FieldData.CreateArray(
177+
"vector",
178+
new double[][]
179+
{
180+
[1, 2],
181+
[3, 4],
182+
});
183+
184+
Assert.Equal(MilvusDataType.Array, field.DataType);
185+
Assert.Equal(MilvusDataType.Double, field.ElementType);
186+
Assert.Equal(2, field.RowCount);
187+
Assert.Equal(2, field.Data.Count);
188+
}
189+
190+
//TODO: differentiate VarChar and String somehow
191+
[Fact]
192+
public void CreateVarCharArrayTest()
193+
{
194+
var field = FieldData.CreateArray(
195+
"vector",
196+
new string[][]
197+
{
198+
["3d4d387208e04a9abe77be65e2b7c7b3", "a5502ddb557047968a70ff69720d2dd2"],
199+
["4c246789a91f4b15aa3b26799df61457", "00a23e95823b4f14854ceed5f7059953"],
200+
});
201+
202+
Assert.Equal(MilvusDataType.Array, field.DataType);
203+
Assert.Equal(MilvusDataType.VarChar, field.ElementType);
204+
Assert.Equal(2, field.RowCount);
205+
Assert.Equal(2, field.Data.Count);
206+
}
70207
}

‎Milvus.Client.sln.DotSettings

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
2+
<s:Int64 x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/BLANK_LINES_AROUND_MULTILINE_CASE_SECTION/@EntryValue">1</s:Int64>
3+
<s:Boolean x:Key="/Default/CodeStyle/CodeFormatting/CSharpFormat/SPACE_WITHIN_SINGLE_LINE_ARRAY_INITIALIZER_BRACES/@EntryValue">True</s:Boolean>
24
<s:Boolean x:Key="/Default/UserDictionary/Words/=Jaccard/@EntryIndexedValue">True</s:Boolean>
35
<s:Boolean x:Key="/Default/UserDictionary/Words/=Milvus/@EntryIndexedValue">True</s:Boolean>
46
<s:Boolean x:Key="/Default/UserDictionary/Words/=pymilvus/@EntryIndexedValue">True</s:Boolean>
57
<s:Boolean x:Key="/Default/UserDictionary/Words/=quantizes/@EntryIndexedValue">True</s:Boolean>
68
<s:Boolean x:Key="/Default/UserDictionary/Words/=quantizing/@EntryIndexedValue">True</s:Boolean>
9+
<s:Boolean x:Key="/Default/UserDictionary/Words/=sbytes/@EntryIndexedValue">True</s:Boolean>
710
<s:Boolean x:Key="/Default/UserDictionary/Words/=Tanimoto/@EntryIndexedValue">True</s:Boolean>
811
<s:Boolean x:Key="/Default/UserDictionary/Words/=topk/@EntryIndexedValue">True</s:Boolean>
912
<s:Boolean x:Key="/Default/UserDictionary/Words/=upserted/@EntryIndexedValue">True</s:Boolean>

‎Milvus.Client/ArrayFieldData.cs

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
namespace Milvus.Client;
2+
3+
/// <summary>
4+
/// Binary Field
5+
/// </summary>
6+
public sealed class ArrayFieldData<TElementData> : FieldData<IReadOnlyList<TElementData>>
7+
{
8+
/// <summary>
9+
/// Construct an array field
10+
/// </summary>
11+
/// <param name="fieldName"></param>
12+
/// <param name="data"></param>
13+
/// <param name="isDynamic"></param>
14+
public ArrayFieldData(string fieldName, IReadOnlyList<IReadOnlyList<TElementData>> data, bool isDynamic)
15+
: base(fieldName, data, MilvusDataType.Array, isDynamic)
16+
{
17+
ElementType = EnsureDataType<TElementData>();
18+
}
19+
20+
/// <summary>
21+
/// Array element type
22+
/// </summary>
23+
public MilvusDataType ElementType { get; }
24+
25+
/// <inheritdoc />
26+
internal override Grpc.FieldData ToGrpcFieldData()
27+
{
28+
Check();
29+
30+
Grpc.FieldData fieldData = new()
31+
{
32+
Type = Grpc.DataType.Array,
33+
IsDynamic = IsDynamic
34+
};
35+
36+
if (FieldName is not null)
37+
{
38+
fieldData.FieldName = FieldName;
39+
}
40+
41+
var arrayArray = new ArrayArray
42+
{
43+
ElementType = (DataType) ElementType,
44+
};
45+
46+
fieldData.Scalars = new ScalarField
47+
{
48+
ArrayData = arrayArray
49+
};
50+
51+
foreach (var array in Data)
52+
{
53+
switch (ElementType)
54+
{
55+
case MilvusDataType.Bool:
56+
BoolArray boolData = new();
57+
boolData.Data.AddRange(array as IEnumerable<bool>);
58+
arrayArray.Data.Add(new ScalarField { BoolData = boolData, });
59+
break;
60+
61+
case MilvusDataType.Int8:
62+
IntArray int8Data = new();
63+
var sbytes = array as IEnumerable<sbyte> ?? Enumerable.Empty<sbyte>();
64+
int8Data.Data.AddRange(sbytes.Select(x => (int) x));
65+
arrayArray.Data.Add(new ScalarField { IntData = int8Data, });
66+
break;
67+
68+
case MilvusDataType.Int16:
69+
IntArray int16Data = new();
70+
var shorts = array as IEnumerable<short> ?? Enumerable.Empty<short>();
71+
int16Data.Data.AddRange(shorts.Select(x => (int) x));
72+
arrayArray.Data.Add(new ScalarField { IntData = int16Data, });
73+
break;
74+
75+
case MilvusDataType.Int32:
76+
IntArray int32Data = new();
77+
int32Data.Data.AddRange(array as IEnumerable<int>);
78+
arrayArray.Data.Add(new ScalarField { IntData = int32Data, });
79+
break;
80+
81+
case MilvusDataType.Int64:
82+
LongArray int64Data = new();
83+
int64Data.Data.AddRange(array as IEnumerable<long>);
84+
arrayArray.Data.Add(new ScalarField { LongData = int64Data, });
85+
break;
86+
87+
case MilvusDataType.Float:
88+
FloatArray floatData = new();
89+
floatData.Data.AddRange(array as IEnumerable<float>);
90+
arrayArray.Data.Add(new ScalarField { FloatData = floatData, });
91+
break;
92+
93+
case MilvusDataType.Double:
94+
DoubleArray doubleData = new();
95+
doubleData.Data.AddRange(array as IEnumerable<double>);
96+
arrayArray.Data.Add(new ScalarField { DoubleData = doubleData, });
97+
break;
98+
99+
case MilvusDataType.String:
100+
StringArray stringData = new();
101+
stringData.Data.AddRange(array as IEnumerable<string>);
102+
arrayArray.Data.Add(new ScalarField { StringData = stringData, });
103+
break;
104+
105+
case MilvusDataType.VarChar:
106+
StringArray varcharData = new();
107+
varcharData.Data.AddRange(array as IEnumerable<string>);
108+
arrayArray.Data.Add(new ScalarField { StringData = varcharData, });
109+
break;
110+
111+
case MilvusDataType.Json:
112+
JSONArray jsonData = new();
113+
var enumerable = array as IEnumerable<string> ?? Enumerable.Empty<string>();
114+
jsonData.Data.AddRange(enumerable.Select(ByteString.CopyFromUtf8));
115+
arrayArray.Data.Add(new ScalarField { JsonData = jsonData, });
116+
break;
117+
118+
case MilvusDataType.None:
119+
throw new MilvusException($"ElementType Error:{DataType}");
120+
121+
default:
122+
throw new MilvusException($"ElementType Error:{DataType}, not supported");
123+
}
124+
}
125+
126+
return fieldData;
127+
}
128+
129+
internal override object GetValueAsObject(int index)
130+
=> ElementType switch
131+
{
132+
MilvusDataType.Bool => ((IReadOnlyList<IEnumerable<bool>>) Data)[index],
133+
MilvusDataType.Int8 => ((IReadOnlyList<IEnumerable<sbyte>>) Data)[index],
134+
MilvusDataType.Int16 => ((IReadOnlyList<IEnumerable<short>>) Data)[index],
135+
MilvusDataType.Int32 => ((IReadOnlyList<IEnumerable<int>>) Data)[index],
136+
MilvusDataType.Int64 => ((IReadOnlyList<IEnumerable<long>>) Data)[index],
137+
MilvusDataType.Float => ((IReadOnlyList<IEnumerable<float>>) Data)[index],
138+
MilvusDataType.Double => ((IReadOnlyList<IEnumerable<double>>) Data)[index],
139+
MilvusDataType.String => ((IReadOnlyList<IEnumerable<string>>) Data)[index],
140+
MilvusDataType.VarChar => ((IReadOnlyList<IEnumerable<string>>) Data)[index],
141+
142+
MilvusDataType.None => throw new MilvusException($"DataType Error:{DataType}"),
143+
_ => throw new MilvusException($"DataType Error:{DataType}, not supported")
144+
};
145+
}

‎Milvus.Client/Constants.cs

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,11 @@ internal static class Constants
7070
/// </summary>
7171
internal const string FailedReason = "failed_reason";
7272

73+
/// <summary>
74+
/// Key name.
75+
/// </summary>
76+
internal const string MaxCapacity = "max_capacity";
77+
7378
/// <summary>
7479
/// Files.
7580
/// </summary>

0 commit comments

Comments
 (0)