From 9f46da26d171e6d49a3548f18e8764590fdd8522 Mon Sep 17 00:00:00 2001 From: ryjiang Date: Wed, 6 Nov 2024 15:36:10 +0800 Subject: [PATCH] [2.5] support default value and null (#369) * init 2.5 Signed-off-by: ryjiang * update milvus version Signed-off-by: ryjiang * update proto Signed-off-by: ryjiang * WIP: support default value & null Signed-off-by: ryjiang * fix json format Signed-off-by: ryjiang * WIP Signed-off-by: ryjiang * WIP Signed-off-by: ryjiang * WIP Signed-off-by: ryjiang * add null test for array Signed-off-by: ryjiang * recover ip Signed-off-by: ryjiang * WIP Signed-off-by: shanghaikid * finish default_null Signed-off-by: ryjiang * update test version Signed-off-by: ryjiang * fix test Signed-off-by: ryjiang * update test Signed-off-by: ryjiang --------- Signed-off-by: ryjiang Signed-off-by: shanghaikid --- milvus/grpc/Collection.ts | 13 +++-- milvus/grpc/Data.ts | 40 +++++++++---- milvus/types/Collection.ts | 9 +-- milvus/types/Data.ts | 9 ++- milvus/utils/Format.ts | 26 ++++++--- milvus/utils/Function.ts | 9 +++ milvus/utils/Grpc.ts | 4 +- package.json | 2 +- proto | 2 +- test/build/Collection.spec.ts | 6 +- test/grpc/Data.spec.ts | 32 ++++++++++- test/grpc/DynamicSchema.spec.ts | 94 +++++++++++++++++++++++-------- test/grpc/Functions.spec.ts | 2 +- test/grpc/MultipleVectors.spec.ts | 2 +- test/grpc/PartitionKey.spec.ts | 4 +- test/tools/collection.ts | 20 +++++-- test/tools/const.ts | 3 +- test/tools/data.ts | 5 +- test/utils/Format.spec.ts | 26 +++++---- test/utils/Function.spec.ts | 28 +++++++++ 20 files changed, 255 insertions(+), 81 deletions(-) diff --git a/milvus/grpc/Collection.ts b/milvus/grpc/Collection.ts index b76d1700..44cfc2b5 100644 --- a/milvus/grpc/Collection.ts +++ b/milvus/grpc/Collection.ts @@ -54,6 +54,7 @@ import { CreateCollectionWithFieldsReq, CreateCollectionWithSchemaReq, FieldSchema, + isVectorType, } from '../'; /** @@ -1054,9 +1055,12 @@ export class Collection extends Database { * }); * ``` */ - async getPkFieldName(data: DescribeCollectionReq): Promise { + async getPkFieldName( + data: DescribeCollectionReq, + desc?: DescribeCollectionResponse + ): Promise { // get collection info - const collectionInfo = await this.describeCollection(data); + const collectionInfo = desc ? desc : await this.describeCollection(data); // pk field let pkField = ''; @@ -1094,10 +1098,11 @@ export class Collection extends Database { * ``` */ async getPkFieldType( - data: DescribeCollectionReq + data: DescribeCollectionReq, + desc?: DescribeCollectionResponse ): Promise { // get collection info - const collectionInfo = await this.describeCollection(data); + const collectionInfo = desc ? desc : await this.describeCollection(data); // pk field type let pkFieldType: keyof typeof DataType = 'Int64'; diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts index c53ba45d..98aee2e3 100644 --- a/milvus/grpc/Data.ts +++ b/milvus/grpc/Data.ts @@ -48,7 +48,7 @@ import { buildDynamicRow, buildFieldDataMap, getDataKey, - Field, + _Field, buildFieldData, BinaryVector, RowData, @@ -65,6 +65,7 @@ import { sparseRowsToBytes, getSparseDim, f32ArrayToBinaryBytes, + getValidDataArray, NO_LIMIT, } from '../'; import { Collection } from './Collection'; @@ -143,7 +144,7 @@ export class Data extends Collection { // Tip: The field data sequence needs to be set same as `collectionInfo.schema.fields`. // If primarykey is set `autoid = true`, you cannot insert the data. // and if function field is set, you need to ignore the field value in the data. - const fieldMap = new Map( + const fieldMap = new Map( collectionInfo.schema.fields .filter(v => !v.is_primary_key || !v.autoID) .filter(v => !v.is_function_output) @@ -155,6 +156,8 @@ export class Data extends Collection { elementType: v.element_type, dim: Number(findKeyValue(v.type_params, 'dim')), data: [], // values container + nullable: v.nullable, + default_value: v.default_value, }, ]) ); @@ -167,6 +170,7 @@ export class Data extends Collection { type: 'JSON', elementType: 'None', data: [], // value container + nullable: false, }); } @@ -225,6 +229,10 @@ export class Data extends Collection { const dataKey = getDataKey(type); const elementType = DataTypeMap[field.elementType!]; const elementTypeKey = getDataKey(elementType); + const valid_data = getValidDataArray( + field.data, + data.fields_data?.length! + ); // build key value let keyValue; @@ -263,14 +271,16 @@ export class Data extends Collection { case DataType.Array: keyValue = { [dataKey]: { - data: field.data.map(d => { - return { - [elementTypeKey]: { - type: elementType, - data: d, - }, - }; - }), + data: field.data + .filter(v => v !== undefined) + .map(d => { + return { + [elementTypeKey]: { + type: elementType, + data: d, + }, + }; + }), element_type: elementType, }, }; @@ -278,17 +288,24 @@ export class Data extends Collection { default: keyValue = { [dataKey]: { - data: field.data, + data: field.data.filter(v => v !== undefined), }, }; break; } + const needValidData = + key !== 'vectors' && + (field.nullable === true || + (typeof field.default_value !== 'undefined' && + field.default_value !== null)); + return { type, field_name: field.name, is_dynamic: field.name === DEFAULT_DYNAMIC_FIELD, [key]: keyValue, + valid_data: needValidData ? valid_data : [], }; }); @@ -931,6 +948,7 @@ export class Data extends Collection { // always get output_fields from fields_data const output_fields = promise.fields_data.map(f => f.field_name); + // build field data map const fieldsDataMap = buildFieldDataMap( promise.fields_data, data.transformers diff --git a/milvus/types/Collection.ts b/milvus/types/Collection.ts index f73ebd1c..9f5084f2 100644 --- a/milvus/types/Collection.ts +++ b/milvus/types/Collection.ts @@ -30,11 +30,11 @@ export interface FieldSchema { element_type?: keyof typeof DataType; default_value?: number | string; dataType: DataType; - is_partition_key: boolean; - is_dynamic: boolean; - is_clustering_key: boolean; + is_partition_key?: boolean; + is_dynamic?: boolean; + is_clustering_key?: boolean; + nullable?: boolean; is_function_output: boolean; - nullable: boolean; } export interface CollectionData { @@ -80,6 +80,7 @@ export interface FieldType { max_capacity?: TypeParam; max_length?: TypeParam; default_value?: number | string; + nullable?: boolean; enable_match?: boolean; tokenizer_params?: Record; enable_tokenizer?: boolean; diff --git a/milvus/types/Data.ts b/milvus/types/Data.ts index b16b6b66..fd817ce4 100644 --- a/milvus/types/Data.ts +++ b/milvus/types/Data.ts @@ -72,19 +72,22 @@ export type FieldData = | VarChar | JSON | Array - | VectorTypes; + | VectorTypes + | null; // Represents a row of data in Milvus. export interface RowData { [x: string]: FieldData; } -export interface Field { +export interface _Field { name: string; type: keyof typeof DataType; elementType?: keyof typeof DataType; data: FieldData[]; dim?: number; + nullable?: boolean; + default_value?: FieldData; } export interface FlushReq extends GrpcTimeOut { @@ -455,6 +458,8 @@ export interface QueryRes extends resStatusResponse { [x: string]: any; data: string; }; + is_dynamic: boolean; + valid_data: boolean[]; }[]; output_fields: string[]; collection_name: string; diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts index 44b7c14d..91c9d071 100644 --- a/milvus/utils/Format.ts +++ b/milvus/utils/Format.ts @@ -10,7 +10,7 @@ import { DescribeCollectionResponse, getDataKey, RowData, - Field, + _Field, JSON, FieldData, CreateCollectionWithFieldsReq, @@ -421,7 +421,7 @@ export const formatDescribedCol = ( */ export const buildDynamicRow = ( rowData: RowData, - fieldMap: Map, + fieldMap: Map, dynamicFieldName: string ) => { const originRow = cloneObj(rowData); @@ -537,17 +537,27 @@ export const buildFieldDataMap = ( }); } - // decode json switch (dataKey) { + // decode json case 'json_data': field_data.forEach((buffer: any, i: number) => { - // console.log(JSON.parse(buffer.toString())); - field_data[i] = JSON.parse(buffer.toString()); + field_data[i] = buffer.length + ? JSON.parse(buffer.toString()) + : null; }); break; default: break; } + + // set the field data with null if item.valid_data is not empty array, it the item in valid_data is false, set the field data with null + if (item.valid_data && item.valid_data.length) { + item.valid_data.forEach((v: any, i: number) => { + if (!v) { + field_data[i] = null; + } + }); + } } // Add the parsed data to the fieldsDataMap @@ -591,7 +601,7 @@ export const getAuthString = (data: { */ export const buildFieldData = ( rowData: RowData, - field: Field, + field: _Field, transformers?: InsertTransformers ): FieldData => { const { type, elementType, name } = field; @@ -614,7 +624,9 @@ export const buildFieldData = ( ? f16Transformer(rowData[name] as Float16Vector) : rowData[name]; case DataType.JSON: - return Buffer.from(JSON.stringify(rowData[name] || {})); + return rowData[name] + ? Buffer.from(JSON.stringify(rowData[name] || {})) + : Buffer.alloc(0); case DataType.Array: const elementField = { ...field, type: elementType! }; return buildFieldData(rowData, elementField, transformers); diff --git a/milvus/utils/Function.ts b/milvus/utils/Function.ts index aeada05c..c3bf4756 100644 --- a/milvus/utils/Function.ts +++ b/milvus/utils/Function.ts @@ -7,6 +7,7 @@ import { DEFAULT_MIN_INT64, SearchResultData, SparseFloatVector, + FieldData, } from '../'; import { Pool } from 'generic-pool'; @@ -236,3 +237,11 @@ export const getSparseDim = (data: SparseFloatVector[]) => { } return dim; }; + +// get valid data +// create a length array with valid data, if the data is undefined or null, return false, otherwise return true +export const getValidDataArray = (data: FieldData[], length: number) => { + return Array.from({ length }).map((_, i) => { + return data[i] !== undefined && data[i] !== null; + }); +}; diff --git a/milvus/utils/Grpc.ts b/milvus/utils/Grpc.ts index 7a450c92..28459b89 100644 --- a/milvus/utils/Grpc.ts +++ b/milvus/utils/Grpc.ts @@ -200,7 +200,7 @@ export const getRetryInterceptor = ({ logger.debug( `\x1b[32m[Response(${ Date.now() - startTime.getTime() - }ms)]\x1b[0m\x1b[2m${clientId}\x1b[0m>${dbname}>\x1b[1m${methodName}\x1b[0m: ${string}` + }ms)]\x1b[0m\x1b[2m${clientId}\x1b[0m>${dbname}>\x1b[1m${methodName}\x1b[0m: ${msg}` ); savedMessageNext(savedReceiveMessage); @@ -217,7 +217,7 @@ export const getRetryInterceptor = ({ const msg = string.length > 2048 ? string.slice(0, 2048) + '...' : string; logger.debug( - `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${string}` + `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${msg}` ); savedSendMessage = message; next(message); diff --git a/package.json b/package.json index aa71e8c8..e28c53ef 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@zilliz/milvus2-sdk-node", "author": "ued@zilliz.com", - "milvusVersion": "master-20241024-f78f6112-amd64", + "milvusVersion": "master-20241105-bd04cac4-amd64", "version": "2.4.9", "main": "dist/milvus", "files": [ diff --git a/proto b/proto index 85ccff4d..7e04a1bc 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit 85ccff4d57fe9c510c88ee4eaf1ba33ef4ef1188 +Subproject commit 7e04a1bcb0d6c20d33bf7353c4dde0fea93b34d6 diff --git a/test/build/Collection.spec.ts b/test/build/Collection.spec.ts index 0df9f9d0..9fc91428 100644 --- a/test/build/Collection.spec.ts +++ b/test/build/Collection.spec.ts @@ -91,7 +91,11 @@ describe('Collection Api', () => { try { await milvusClient.createCollection( - genCollectionParams({ collectionName: 'any', dim: [8] }) + genCollectionParams({ + collectionName: 'any', + vectorType: [DataType.BinaryVector], + dim: [6], + }) ); } catch (error) { expect(error.message).toEqual( diff --git a/test/grpc/Data.spec.ts b/test/grpc/Data.spec.ts index ce63aeda..a0d18f82 100644 --- a/test/grpc/Data.spec.ts +++ b/test/grpc/Data.spec.ts @@ -19,6 +19,7 @@ import { timeoutTest } from '../tools'; const milvusClient = new MilvusClient({ address: IP, + logLevel: 'info', }); const COLLECTION_NAME = GENERATE_NAME(); const VARCHAR_ID_COLLECTION_NAME = GENERATE_NAME(); @@ -30,6 +31,14 @@ const createCollectionParams = genCollectionParams({ dim: [4], vectorType: [DataType.FloatVector], autoID: false, + fields: [ + { + name: 'varChar2', + description: 'VarChar2 field', + data_type: DataType.VarChar, + max_length: 100, + }, + ], }); const createCollectionParamsVarcharID = genCollectionParams({ collectionName: VARCHAR_ID_COLLECTION_NAME, @@ -61,6 +70,7 @@ describe(`Data.API`, () => { collection_name: COLLECTION_NAME, data: generateInsertData(createCollectionParams.fields, 1024), }); + await milvusClient.insert({ collection_name: VARCHAR_ID_COLLECTION_NAME, data: generateInsertData(createCollectionParamsVarcharID.fields, 1024), @@ -100,6 +110,16 @@ describe(`Data.API`, () => { }); afterAll(async () => { + const searchParams = { + collection_name: COLLECTION_NAME, + // partition_names: [], + filter: 'json["number"] >= 0', + data: [1, 2, 3, 4], + limit: 4, + output_fields: ['id', 'json'], + }; + const res = await milvusClient.search(searchParams); + await milvusClient.dropCollection({ collection_name: COLLECTION_NAME, }); @@ -231,13 +251,21 @@ describe(`Data.API`, () => { it(`Exec simple search without params and output fields should success`, async () => { const limit = 4; - // collection search + const describe = await milvusClient.describeCollection({ + collection_name: COLLECTION_NAME, + }); + + // find varchar2 field + describe.schema.fields.find(f => f.name === 'varChar2'); + + // console.dir(varChar2Field, { depth: null }); + const searchWithData = await milvusClient.search({ collection_name: COLLECTION_NAME, filter: '', data: [1, 2, 3, 4], limit: limit, - group_by_field: 'varChar', + group_by_field: 'varChar2', }); expect(searchWithData.status.error_code).toEqual(ErrorCode.SUCCESS); diff --git a/test/grpc/DynamicSchema.spec.ts b/test/grpc/DynamicSchema.spec.ts index 64bebe98..530a08f9 100644 --- a/test/grpc/DynamicSchema.spec.ts +++ b/test/grpc/DynamicSchema.spec.ts @@ -4,6 +4,7 @@ import { ErrorCode, ConsistencyLevelEnum, } from '../../milvus'; +import { DEFAULT_STRING_VALUE, DEFAULT_NUM_VALUE } from '../tools/const'; import { IP, genCollectionParams, @@ -12,7 +13,7 @@ import { dynamicFields, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP }); +const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' }); const COLLECTION = GENERATE_NAME(); const dbParam = { db_name: 'DynamicSchema', @@ -25,7 +26,7 @@ const createCollectionParams = genCollectionParams({ dim: [4], vectorType: [DataType.FloatVector], autoID: false, - partitionKeyEnabled: true, + partitionKeyEnabled: false, numPartitions, enableDynamic: true, }); @@ -58,7 +59,7 @@ describe(`Dynamic schema API`, () => { it(`Insert data with dynamic field should success`, async () => { const data = generateInsertData( [...createCollectionParams.fields, ...dynamicFields], - 20 + 10 ); const insert = await milvusClient.insert({ @@ -100,43 +101,90 @@ describe(`Dynamic schema API`, () => { 'json', 'vector', 'id', + 'float', + 'bool', + 'default_value', + 'varChar', + 'json', 'dynamic_int64', 'dynamic_varChar', + 'int32_array', ], - consistency_level: ConsistencyLevelEnum.Session, + consistency_level: ConsistencyLevelEnum.Strong, }); expect(query.status.error_code).toEqual(ErrorCode.SUCCESS); expect(query.data.length).toEqual(10); + // get test values + const varChars = query.data.map(v => v.varChar); + const defaultValues = query.data.map(v => v.default_value); + const jsons = query.data.map(v => v.json); + const arrays = query.data.map(v => v.int32_array); + const bools = query.data.map(v => v.bool); + const floats = query.data.map(v => v.float); + // some of floats should be equal to DEFAULT_NUM_VALUE + expect(floats.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual(true); + // some of varChar should be equal to DEFAULT_STRING_VALUE + expect(varChars.some(v => v === DEFAULT_STRING_VALUE)).toEqual(true); + // some of default_value should be equal to DEFAULT_NUM_VALUE + expect(defaultValues.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual( + true + ); + // some of json should be null + expect(jsons.some(v => v === null)).toEqual(true); + // some of bools should be null + expect(bools.some(v => v === null)).toEqual(true); + // some of array should be null + expect(arrays.some(v => v === null)).toEqual(true); }); - it(`search with dynamic field should success`, async () => { + // it(`query null should success`, async () => { + // // query + // const query = await milvusClient.query({ + // collection_name: COLLECTION, + // limit: 10, + // expr: 'ISNULL(float)', + // output_fields: ['float'], + // consistency_level: ConsistencyLevelEnum.Strong, + // }); + + // console.dir(query, { depth: null }); + // }); + + it(`search with null field should success`, async () => { // search const search = await milvusClient.search({ collection_name: COLLECTION, limit: 10, - data: [ - [1, 2, 3, 4], - [1, 2, 3, 4], - ], + data: [1, 2, 3, 4], expr: 'id > 0', output_fields: ['*'], - consistency_level: ConsistencyLevelEnum.Session, + consistency_level: ConsistencyLevelEnum.Strong, }); expect(search.status.error_code).toEqual(ErrorCode.SUCCESS); - expect(search.results.length).toEqual(2); - expect(search.results[0].length).toEqual(10); - - // search - const search2 = await milvusClient.search({ - collection_name: COLLECTION, - limit: 10, - data: [1, 2, 3, 4], - expr: 'id > 0', - output_fields: ['json', 'id', 'dynamic_int64', 'dynamic_varChar'], - }); - expect(search2.status.error_code).toEqual(ErrorCode.SUCCESS); - expect(search2.results.length).toEqual(10); + expect(search.results.length).toEqual(10); + + // get test values + const varChars = search.results.map(v => v.varChar); + const defaultValues = search.results.map(v => v.default_value); + const jsons = search.results.map(v => v.json); + const arrays = search.results.map(v => v.int32_array); + const bools = search.results.map(v => v.bool); + const floats = search.results.map(v => v.float); + // some of floats should be equal to DEFAULT_NUM_VALUE + expect(floats.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual(true); + // some of varChar should be equal to DEFAULT_STRING_VALUE + expect(varChars.some(v => v === DEFAULT_STRING_VALUE)).toEqual(true); + // some of default_value should be equal to DEFAULT_NUM_VALUE + expect(defaultValues.some(v => Number(v) === DEFAULT_NUM_VALUE)).toEqual( + true + ); + // some of json should be null + expect(jsons.some(v => v === null)).toEqual(true); + // some of bools should be null + expect(bools.some(v => v === null)).toEqual(true); + // some of array should be null + expect(arrays.some(v => v === null)).toEqual(true); }); }); diff --git a/test/grpc/Functions.spec.ts b/test/grpc/Functions.spec.ts index f68c1301..d9ee95f2 100644 --- a/test/grpc/Functions.spec.ts +++ b/test/grpc/Functions.spec.ts @@ -14,7 +14,7 @@ import { dynamicFields, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP, logLevel: 'debug' }); +const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' }); const COLLECTION = GENERATE_NAME(); const dbParam = { db_name: 'Functions', diff --git a/test/grpc/MultipleVectors.spec.ts b/test/grpc/MultipleVectors.spec.ts index d31665c2..f8f24378 100644 --- a/test/grpc/MultipleVectors.spec.ts +++ b/test/grpc/MultipleVectors.spec.ts @@ -16,7 +16,7 @@ import { generateInsertData, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP, logLevel: 'debug' }); +const milvusClient = new MilvusClient({ address: IP, logLevel: 'info' }); const COLLECTION_NAME = GENERATE_NAME(); const dbParam = { diff --git a/test/grpc/PartitionKey.spec.ts b/test/grpc/PartitionKey.spec.ts index aa4f53cd..c05d6156 100644 --- a/test/grpc/PartitionKey.spec.ts +++ b/test/grpc/PartitionKey.spec.ts @@ -12,7 +12,7 @@ import { generateInsertData, } from '../tools'; -const milvusClient = new MilvusClient({ address: IP }); +const milvusClient = new MilvusClient({ address: IP , logLevel: 'info' }); const COLLECTION_NAME = GENERATE_NAME(); const COLLECTION_NAME2 = GENERATE_NAME(); const COLLECTION_NAME3 = GENERATE_NAME(); @@ -132,7 +132,7 @@ describe(`Partition key API`, () => { it(`it should create collection successfully with partition_key_field set`, async () => { const createCollectionParams = genCollectionParams({ - collectionName: COLLECTION_NAME2, + collectionName: COLLECTION_NAME3, dim: [4], vectorType: [DataType.FloatVector], autoID: false, diff --git a/test/tools/collection.ts b/test/tools/collection.ts index 83902ffb..a44ef4a0 100644 --- a/test/tools/collection.ts +++ b/test/tools/collection.ts @@ -1,10 +1,14 @@ +import { + MAX_CAPACITY, + MAX_LENGTH, + DEFAULT_NUM_VALUE, + DEFAULT_STRING_VALUE, +} from './const'; import { DataType, ConsistencyLevelEnum, - FunctionType, Function, } from '../../milvus'; -import { MAX_CAPACITY, MAX_LENGTH } from './const'; import { GENERATE_VECTOR_NAME } from './'; export const dynamicFields = [ @@ -105,23 +109,27 @@ export const genCollectionParams = (data: { { name: 'float', description: 'Float field', + default_value: DEFAULT_NUM_VALUE, data_type: DataType.Float, }, { name: 'bool', description: 'bool field', + nullable: true, data_type: DataType.Bool, }, { name: 'default_value', - // default_value: DEFAULT_VALUE, + nullable: true, + default_value: DEFAULT_NUM_VALUE, description: 'int64 field', - data_type: 'Int64', // test string data type + data_type: 'Int64', // }, { name: 'varChar', description: 'VarChar field', data_type: DataType.VarChar, + default_value: DEFAULT_STRING_VALUE, max_length: MAX_LENGTH, is_partition_key: partitionKeyEnabled, enable_tokenizer: true, @@ -129,13 +137,15 @@ export const genCollectionParams = (data: { { name: 'json', description: 'JSON field', + nullable: true, data_type: DataType.JSON, }, { name: 'int32_array', description: 'int array field', data_type: DataType.Array, - element_type: 'Int32', // test string element type + nullable: true, + element_type: 'Int32', max_capacity: maxCapacity || MAX_CAPACITY, }, { diff --git a/test/tools/const.ts b/test/tools/const.ts index a63c2adb..5ddde9f5 100644 --- a/test/tools/const.ts +++ b/test/tools/const.ts @@ -3,7 +3,8 @@ export const INDEX_NAME = 'index_name'; export const DIMENSION = 4; export const INDEX_FILE_SIZE = 1024; export const PARTITION_TAG = 'random'; -export const DEFAULT_VALUE = '100'; +export const DEFAULT_NUM_VALUE = 100; +export const DEFAULT_STRING_VALUE = 'd'; export const MAX_LENGTH = 8; export const MAX_CAPACITY = 4; export const VECTOR_FIELD_NAME = 'vector'; diff --git a/test/tools/data.ts b/test/tools/data.ts index b0c8e07b..16004a46 100644 --- a/test/tools/data.ts +++ b/test/tools/data.ts @@ -283,15 +283,14 @@ export const generateInsertData = ( for (const field of fields) { // Skip autoID and fields with default values - if (field.autoID || typeof field.default_value !== 'undefined') { + if (field.autoID) { continue; } // get data type const data_type = convertToDataType(field.data_type); - // Skip fields with default values - if (typeof field.default_value !== 'undefined') { + if ((field.nullable || field.default_value) && Math.random() < 0.5) { continue; } diff --git a/test/utils/Format.spec.ts b/test/utils/Format.spec.ts index f31c57ca..dc0ad36f 100644 --- a/test/utils/Format.spec.ts +++ b/test/utils/Format.spec.ts @@ -25,7 +25,7 @@ import { getAuthString, buildFieldData, formatSearchResult, - Field, + _Field, formatSearchData, buildSearchRequest, FieldSchema, @@ -467,7 +467,7 @@ describe('utils/format', () => { name: 'key1', type: 'VarChar', data: [{ key1: 'value1' }], - } as Field, + } as _Field, ], ]); const dynamicField = 'dynamic'; @@ -487,7 +487,7 @@ describe('utils/format', () => { name: 'key1', type: 'VarChar', data: [{ key1: 'value1' }], - } as Field, + } as _Field, ], [ 'key2', @@ -495,7 +495,7 @@ describe('utils/format', () => { name: 'key2', type: 'VarChar', data: [{ key2: 'value2' }], - } as Field, + } as _Field, ], ]); const dynamicField = 'dynamic'; @@ -516,7 +516,7 @@ describe('utils/format', () => { name: 'key1', type: 'VarChar', data: [{ key1: 'value1' }], - } as Field, + } as _Field, ], ]); const dynamicField = 'dynamic'; @@ -548,31 +548,37 @@ describe('utils/format', () => { it('should return the value of the field for BinaryVector and FloatVector types', () => { const row = { name: 'John', vector: [1, 2, 3] }; const field = { type: 'BinaryVector', name: 'vector' }; - expect(buildFieldData(row, field as Field)).toEqual([1, 2, 3]); + expect(buildFieldData(row, field as _Field)).toEqual([1, 2, 3]); field.type = 'FloatVector'; - expect(buildFieldData(row, field as Field)).toEqual([1, 2, 3]); + expect(buildFieldData(row, field as _Field)).toEqual([1, 2, 3]); }); it('should return the JSON stringified value of the field for JSON type', () => { const row = { name: 'John', data: { age: 25, city: 'New York' } }; const field = { type: 'JSON', name: 'data' }; - expect(JSON.parse(buildFieldData(row, field as Field).toString())).toEqual({ + expect( + JSON.parse(buildFieldData(row, field as _Field)!.toString()) + ).toEqual({ age: 25, city: 'New York', }); + + // if json field is not in the row, should return Buffer.alloc(0) + const row2 = { name: 'John' }; + expect(buildFieldData(row2, field as _Field)).toEqual(Buffer.alloc(0)); }); it('should recursively call buildFieldData for Array type', () => { const row = { name: 'John', array: [1, 2, 3] }; const field = { type: 'Array', elementType: 'Int', name: 'array' }; - expect(buildFieldData(row, field as Field)).toEqual([1, 2, 3]); + expect(buildFieldData(row, field as _Field)).toEqual([1, 2, 3]); }); it('should return the value of the field for other types', () => { const row = { name: 'John', age: 25 }; const field = { type: 'Int', name: 'age' }; - expect(buildFieldData(row, field as Field)).toEqual(25); + expect(buildFieldData(row, field as _Field)).toEqual(25); }); it('should format search results correctly', () => { diff --git a/test/utils/Function.spec.ts b/test/utils/Function.spec.ts index d742657a..035fca5d 100644 --- a/test/utils/Function.spec.ts +++ b/test/utils/Function.spec.ts @@ -10,6 +10,7 @@ import { SparseFloatVector, getDataKey, DataType, + getValidDataArray, } from '../../milvus'; describe('Function API testing', () => { @@ -349,4 +350,31 @@ describe('Function API testing', () => { expect(getDataKey(DataType.JSON, true)).toEqual('jsonData'); expect(getDataKey(DataType.None, true)).toEqual('none'); }); + + it('should return the valid array', () => { + const a = [1, 2, 3]; + const length = 5; + const result = getValidDataArray(a, length); + expect(result).toEqual([true, true, true, false, false]); + + const b = [1, null, 3]; + const result2 = getValidDataArray(b, length); + expect(result2).toEqual([true, false, true, false, false]); + + const c: any = []; + const result3 = getValidDataArray(c, length); + expect(result3).toEqual([false, false, false, false, false]); + + const d: any = [1, 2, 3, 4, undefined]; + const result4 = getValidDataArray(d, length); + expect(result4).toEqual([true, true, true, true, false]); + + const e = [ + [1, 2], + [3, 4], + [5, 6], + ]; + const result5 = getValidDataArray(e, length); + expect(result5).toEqual([true, true, true, false, false]); + }); });