support insert null (#382)

* avoid using type Function Signed-off-by: ryjiang <jiangruiyi@gmail.com> * support insert null Signed-off-by: ryjiang <jiangruiyi@gmail.com> * fix build Signed-off-by: ryjiang <jiangruiyi@gmail.com> * fix test Signed-off-by: ryjiang <jiangruiyi@gmail.com> * update grpc util Signed-off-by: ryjiang <jiangruiyi@gmail.com> --------- Signed-off-by: ryjiang <jiangruiyi@gmail.com>
milvus-io · Dec 11, 2024 · 22e613e · 22e613e
1 parent 0f6fe26
commit 22e613e
Show file tree

Hide file tree

Showing 9 changed files with 42 additions and 23 deletions.
diff --git a/milvus/grpc/Data.ts b/milvus/grpc/Data.ts
@@ -212,9 +212,7 @@ export class Data extends Collection {
         switch (DataTypeMap[field.type]) {
           case DataType.BinaryVector:
           case DataType.FloatVector:
-            field.data = (field.data as number[]).concat(
-              buildFieldData(rowData, field) as number[]
-            );
+            field.data = field.data.concat(buildFieldData(rowData, field));
             break;
           default:
             field.data[rowIndex] = buildFieldData(

diff --git a/milvus/types/Collection.ts b/milvus/types/Collection.ts
@@ -92,7 +92,7 @@ export interface ShowCollectionsReq extends GrpcTimeOut {
 
 export type Properties = Record<string, string | number | boolean>;
 
-export type Function = {
+export type FunctionObject = {
   name: string;
   description?: string;
   type: FunctionType;
@@ -119,7 +119,7 @@ export interface BaseCreateCollectionReq extends GrpcTimeOut {
   enableDynamicField?: boolean; // optional, alias of enable_dynamic_field
   properties?: Properties; // optional, collection properties
   db_name?: string; // optional, db name
-  functions?: Function[]; // optionals, doc-in/doc-out functions
+  functions?: FunctionObject[]; // optionals, doc-in/doc-out functions
 }
 
 export interface CreateCollectionWithFieldsReq extends BaseCreateCollectionReq {
@@ -203,7 +203,7 @@ export interface CollectionSchema {
   enable_dynamic_field: boolean;
   autoID: boolean;
   fields: FieldSchema[];
-  functions: Function[];
+  functions: FunctionObject[];
 }
 
 export interface DescribeCollectionResponse extends TimeStamp {
@@ -222,7 +222,7 @@ export interface DescribeCollectionResponse extends TimeStamp {
   shards_num: number;
   num_partitions?: string; // int64
   db_name: string;
-  functions: Function[];
+  functions: FunctionObject[];
 }
 
 export interface GetCompactionPlansResponse extends resStatusResponse {

diff --git a/milvus/types/Data.ts b/milvus/types/Data.ts
@@ -73,7 +73,8 @@ export type FieldData =
   | JSON
   | Array
   | VectorTypes
-  | null;
+  | null
+  | undefined;
 
 // Represents a row of data in Milvus.
 export interface RowData {
@@ -107,14 +108,28 @@ export type InsertTransformers = {
   [DataType.Float16Vector]?: (f16: Float16Vector) => Buffer;
 };
 
-export interface InsertReq extends collectionNameReq {
+// Base properties shared by both variants
+interface BaseInsertReq extends collectionNameReq {
   partition_name?: string; // partition name
-  data?: RowData[]; // data to insert
-  fields_data?: RowData[]; // alias for data
-  hash_keys?: Number[]; // user can generate hash value depend on primarykey value
+  hash_keys?: number[]; // user can generate hash value depend on primarykey value
   transformers?: InsertTransformers; // provide custom data transformer for specific data type like bf16 or f16 vectors
 }
 
+// Variant with data property
+interface DataInsertReq extends BaseInsertReq {
+  data: RowData[]; // data to insert
+  fields_data?: never; // Ensure fields_data cannot be used
+}
+
+// Variant with fields_data property
+interface FieldsDataInsertReq extends BaseInsertReq {
+  fields_data: RowData[]; // alias for data
+  data?: never; // Ensure data cannot be used
+}
+
+// Union type to enforce mutual exclusivity
+export type InsertReq = DataInsertReq | FieldsDataInsertReq;
+
 interface BaseDeleteReq extends collectionNameReq {
   partition_name?: string; // partition name
   consistency_level?:

diff --git a/milvus/utils/Format.ts b/milvus/utils/Format.ts
@@ -651,9 +651,11 @@ export const buildFieldData = (
         : Buffer.alloc(0);
     case DataType.Array:
       const elementField = { ...field, type: elementType! };
-      return buildFieldData(rowData, elementField, transformers);
+      return rowData[name] === null
+        ? undefined
+        : buildFieldData(rowData, elementField, transformers);
     default:
-      return rowData[name];
+      return rowData[name] === null ? undefined : rowData[name];
   }
 };
 

diff --git a/milvus/utils/Grpc.ts b/milvus/utils/Grpc.ts
@@ -195,7 +195,7 @@ export const getRetryInterceptor = ({
             } else {
               const string = JSON.stringify(savedReceiveMessage);
               const msg =
-                string.length > 2048 ? string.slice(0, 2048) + '...' : string;
+                string.length > 4096 ? string.slice(0, 4096) + '...' : string;
 
               logger.debug(
                 `\x1b[32m[Response(${
@@ -215,7 +215,7 @@ export const getRetryInterceptor = ({
         const string = JSON.stringify(message);
         // if string is too big, just show 1000 characters
         const msg =
-          string.length > 2048 ? string.slice(0, 2048) + '...' : string;
+          string.length > 4096 ? string.slice(0, 4096) + '...' : string;
         logger.debug(
           `\x1b[34m[Request]\x1b[0m${clientId}>${dbname}>\x1b[1m${methodName}(${timeoutInSeconds})\x1b[0m: ${msg}`
         );

diff --git a/test/grpc/PartialLoad.spec.ts b/test/grpc/PartialLoad.spec.ts
@@ -87,7 +87,7 @@ describe(`Partial load API `, () => {
       expr: `id > 0`,
       output_fields: ['*'],
     });
-    console.dir(query, { depth: null });
+    // console.dir(query, { depth: null });
     expect(query.status.error_code).toEqual(ErrorCode.SUCCESS);
     // result should not contain 'array' field
     const keys = Object.keys(query.data[0]);

diff --git a/test/tools/collection.ts b/test/tools/collection.ts
@@ -7,7 +7,7 @@ import {
 import {
   DataType,
   ConsistencyLevelEnum,
-  Function,
+  FunctionObject,
 } from '../../milvus';
 import { GENERATE_VECTOR_NAME } from './';
 
@@ -51,7 +51,7 @@ export const genCollectionParams = (data: {
   enableDynamic?: boolean;
   maxCapacity?: number;
   idType?: DataType;
-  functions?: Function[];
+  functions?: FunctionObject[];
 }) => {
   const {
     collectionName,

diff --git a/test/tools/data.ts b/test/tools/data.ts
@@ -290,7 +290,11 @@ export const generateInsertData = (
       // get data type
       const data_type = convertToDataType(field.data_type);
 
-      if ((field.nullable || field.default_value) && Math.random() < 0.5) {
+      if (field.nullable && Math.random() < 0.5) {
+        value[field.name] = null;
+        continue;
+      }
+      if (field.default_value && Math.random() < 0.5) {
         continue;
       }
 

diff --git a/test/utils/Format.spec.ts b/test/utils/Format.spec.ts
@@ -562,7 +562,7 @@ describe('utils/format', () => {
   });
 
   it('should return an object with keys from data and fieldsDataMap', () => {
-    const data = { key1: 'value1', key2: 'value2' };
+    const data = { key1: 'value1', key2: null };
     const fieldsDataMap = new Map([
       [
         'key1',
@@ -577,7 +577,7 @@ describe('utils/format', () => {
         {
           name: 'key2',
           type: 'VarChar',
-          data: [{ key2: 'value2' }],
+          data: [{ key2: null }],
         } as _Field,
       ],
     ]);
@@ -586,7 +586,7 @@ describe('utils/format', () => {
     expect(result).toEqual({
       [dynamicField]: {},
       key1: 'value1',
-      key2: 'value2',
+      key2: null,
     });
   });