From 22e613e4c2725b610200ee2deed5a1bd80a10be3 Mon Sep 17 00:00:00 2001 From: ryjiang Date: Wed, 11 Dec 2024 14:44:56 +0800 Subject: [PATCH] support insert null (#382) * avoid using type Function Signed-off-by: ryjiang * support insert null Signed-off-by: ryjiang * fix build Signed-off-by: ryjiang * fix test Signed-off-by: ryjiang * update grpc util Signed-off-by: ryjiang --------- Signed-off-by: ryjiang --- milvus/grpc/Data.ts | 4 +--- milvus/types/Collection.ts | 8 ++++---- milvus/types/Data.ts | 25 ++++++++++++++++++++----- milvus/utils/Format.ts | 6 ++++-- milvus/utils/Grpc.ts | 4 ++-- test/grpc/PartialLoad.spec.ts | 2 +- test/tools/collection.ts | 4 ++-- test/tools/data.ts | 6 +++++- test/utils/Format.spec.ts | 6 +++--- 9 files changed, 42 insertions(+), 23 deletions(-) diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts index c7d5541d..b9a6c435 100644 --- a/milvus/grpc/Data.ts +++ b/milvus/grpc/Data.ts @@ -212,9 +212,7 @@ export class Data extends Collection { switch (DataTypeMap[field.type]) { case DataType.BinaryVector: case DataType.FloatVector: - field.data = (field.data as number[]).concat( - buildFieldData(rowData, field) as number[] - ); + field.data = field.data.concat(buildFieldData(rowData, field)); break; default: field.data[rowIndex] = buildFieldData( diff --git a/milvus/types/Collection.ts b/milvus/types/Collection.ts index 65f9512a..7014e872 100644 --- a/milvus/types/Collection.ts +++ b/milvus/types/Collection.ts @@ -92,7 +92,7 @@ export interface ShowCollectionsReq extends GrpcTimeOut { export type Properties = Record; -export type Function = { +export type FunctionObject = { name: string; description?: string; type: FunctionType; @@ -119,7 +119,7 @@ export interface BaseCreateCollectionReq extends GrpcTimeOut { enableDynamicField?: boolean; // optional, alias of enable_dynamic_field properties?: Properties; // optional, collection properties db_name?: string; // optional, db name - functions?: Function[]; // optionals, doc-in/doc-out functions + functions?: FunctionObject[]; // optionals, doc-in/doc-out functions } export interface CreateCollectionWithFieldsReq extends BaseCreateCollectionReq { @@ -203,7 +203,7 @@ export interface CollectionSchema { enable_dynamic_field: boolean; autoID: boolean; fields: FieldSchema[]; - functions: Function[]; + functions: FunctionObject[]; } export interface DescribeCollectionResponse extends TimeStamp { @@ -222,7 +222,7 @@ export interface DescribeCollectionResponse extends TimeStamp { shards_num: number; num_partitions?: string; // int64 db_name: string; - functions: Function[]; + functions: FunctionObject[]; } export interface GetCompactionPlansResponse extends resStatusResponse { diff --git a/milvus/types/Data.ts b/milvus/types/Data.ts index f36707fc..1f7c26df 100644 --- a/milvus/types/Data.ts +++ b/milvus/types/Data.ts @@ -73,7 +73,8 @@ export type FieldData = | JSON | Array | VectorTypes - | null; + | null + | undefined; // Represents a row of data in Milvus. export interface RowData { @@ -107,14 +108,28 @@ export type InsertTransformers = { [DataType.Float16Vector]?: (f16: Float16Vector) => Buffer; }; -export interface InsertReq extends collectionNameReq { +// Base properties shared by both variants +interface BaseInsertReq extends collectionNameReq { partition_name?: string; // partition name - data?: RowData[]; // data to insert - fields_data?: RowData[]; // alias for data - hash_keys?: Number[]; // user can generate hash value depend on primarykey value + hash_keys?: number[]; // user can generate hash value depend on primarykey value transformers?: InsertTransformers; // provide custom data transformer for specific data type like bf16 or f16 vectors } +// Variant with data property +interface DataInsertReq extends BaseInsertReq { + data: RowData[]; // data to insert + fields_data?: never; // Ensure fields_data cannot be used +} + +// Variant with fields_data property +interface FieldsDataInsertReq extends BaseInsertReq { + fields_data: RowData[]; // alias for data + data?: never; // Ensure data cannot be used +} + +// Union type to enforce mutual exclusivity +export type InsertReq = DataInsertReq | FieldsDataInsertReq; + interface BaseDeleteReq extends collectionNameReq { partition_name?: string; // partition name consistency_level?: diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts index 2a8e63ef..abe5680c 100644 --- a/milvus/utils/Format.ts +++ b/milvus/utils/Format.ts @@ -651,9 +651,11 @@ export const buildFieldData = ( : Buffer.alloc(0); case DataType.Array: const elementField = { ...field, type: elementType! }; - return buildFieldData(rowData, elementField, transformers); + return rowData[name] === null + ? undefined + : buildFieldData(rowData, elementField, transformers); default: - return rowData[name]; + return rowData[name] === null ? undefined : rowData[name]; } }; diff --git a/milvus/utils/Grpc.ts b/milvus/utils/Grpc.ts index 28459b89..4945ca91 100644 --- a/milvus/utils/Grpc.ts +++ b/milvus/utils/Grpc.ts @@ -195,7 +195,7 @@ export const getRetryInterceptor = ({ } else { const string = JSON.stringify(savedReceiveMessage); const msg = - string.length > 2048 ? string.slice(0, 2048) + '...' : string; + string.length > 4096 ? string.slice(0, 4096) + '...' : string; logger.debug( `\x1b[32m[Response(${ @@ -215,7 +215,7 @@ export const getRetryInterceptor = ({ const string = JSON.stringify(message); // if string is too big, just show 1000 characters const msg = - string.length > 2048 ? string.slice(0, 2048) + '...' : string; + string.length > 4096 ? string.slice(0, 4096) + '...' : string; logger.debug( `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${msg}` ); diff --git a/test/grpc/PartialLoad.spec.ts b/test/grpc/PartialLoad.spec.ts index 66fd3058..0658551c 100644 --- a/test/grpc/PartialLoad.spec.ts +++ b/test/grpc/PartialLoad.spec.ts @@ -87,7 +87,7 @@ describe(`Partial load API `, () => { expr: `id > 0`, output_fields: ['*'], }); - console.dir(query, { depth: null }); + // console.dir(query, { depth: null }); expect(query.status.error_code).toEqual(ErrorCode.SUCCESS); // result should not contain 'array' field const keys = Object.keys(query.data[0]); diff --git a/test/tools/collection.ts b/test/tools/collection.ts index 22bb4ef4..fdb54ca3 100644 --- a/test/tools/collection.ts +++ b/test/tools/collection.ts @@ -7,7 +7,7 @@ import { import { DataType, ConsistencyLevelEnum, - Function, + FunctionObject, } from '../../milvus'; import { GENERATE_VECTOR_NAME } from './'; @@ -51,7 +51,7 @@ export const genCollectionParams = (data: { enableDynamic?: boolean; maxCapacity?: number; idType?: DataType; - functions?: Function[]; + functions?: FunctionObject[]; }) => { const { collectionName, diff --git a/test/tools/data.ts b/test/tools/data.ts index 16004a46..b0ac821f 100644 --- a/test/tools/data.ts +++ b/test/tools/data.ts @@ -290,7 +290,11 @@ export const generateInsertData = ( // get data type const data_type = convertToDataType(field.data_type); - if ((field.nullable || field.default_value) && Math.random() < 0.5) { + if (field.nullable && Math.random() < 0.5) { + value[field.name] = null; + continue; + } + if (field.default_value && Math.random() < 0.5) { continue; } diff --git a/test/utils/Format.spec.ts b/test/utils/Format.spec.ts index a350a2ec..d40bae2e 100644 --- a/test/utils/Format.spec.ts +++ b/test/utils/Format.spec.ts @@ -562,7 +562,7 @@ describe('utils/format', () => { }); it('should return an object with keys from data and fieldsDataMap', () => { - const data = { key1: 'value1', key2: 'value2' }; + const data = { key1: 'value1', key2: null }; const fieldsDataMap = new Map([ [ 'key1', @@ -577,7 +577,7 @@ describe('utils/format', () => { { name: 'key2', type: 'VarChar', - data: [{ key2: 'value2' }], + data: [{ key2: null }], } as _Field, ], ]); @@ -586,7 +586,7 @@ describe('utils/format', () => { expect(result).toEqual({ [dynamicField]: {}, key1: 'value1', - key2: 'value2', + key2: null, }); });