Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

replace axios with node-fetch #254

Merged
merged 9 commits into from
Nov 1, 2023
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
64 changes: 51 additions & 13 deletions milvus/HttpClient.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import fetch from 'node-fetch';
import { HttpClientConfig } from './types';
import { Collection, Vector } from './http';
import {
Expand All @@ -7,14 +6,42 @@ import {
DEFAULT_HTTP_ENDPOINT_VERSION,
} from '../milvus/const';

// base class
/**
* HttpBaseClient is a base class for making HTTP requests to a Milvus server.
* It provides basic functionality for making GET and POST requests, and handles
* configuration, headers, and timeouts.
*
* The HttpClientConfig object should contain the following properties:
* - endpoint: The URL of the Milvus server.
* - username: (Optional) The username for authentication.
* - password: (Optional) The password for authentication.
* - token: (Optional) The token for authentication.
* - fetch: (Optional) An alternative fetch API implementation, e.g., node-fetch for Node.js environments.
* - baseURL: (Optional) The base URL for the API endpoints.
* - version: (Optional) The version of the API endpoints.
* - database: (Optional) The default database to use for requests.
* - timeout: (Optional) The timeout for requests in milliseconds.
*
* Note: This is a base class and does not provide specific methods for interacting
* with Milvus entities like collections or vectors. For that, use the HttpClient class
* which extends this class and mixes in the Collection and Vector APIs.
*/
export class HttpBaseClient {
// The client configuration.
public config: HttpClientConfig;

constructor(config: HttpClientConfig) {
shanghaikid marked this conversation as resolved.
Show resolved Hide resolved
// Assign the configuration object.
this.config = config;

// The fetch method used for requests can be customized by providing a fetch property in the configuration.
// If no fetch method is provided, the global fetch method will be used if available.
// If no global fetch method is available, an error will be thrown.
if (!this.config.fetch && typeof fetch === 'undefined') {
throw new Error(
'The Fetch API is not supported in this environment. Please provide an alternative, for example, node-fetch.'
);
}
}

// baseURL
Expand Down Expand Up @@ -48,6 +75,7 @@ export class HttpBaseClient {
return this.config.timeout || DEFAULT_HTTP_TIMEOUT;
}

// headers
get headers() {
return {
Authorization: this.authorization,
Expand All @@ -56,18 +84,24 @@ export class HttpBaseClient {
};
}

// fetch
get fetch() {
return this.config.fetch || fetch;
shanghaikid marked this conversation as resolved.
Show resolved Hide resolved
}

// POST API
async POST<T>(url: string, data: Record<string, any> = {}): Promise<T> {
try {
// timeout controller
const controller = new AbortController();
shanghaikid marked this conversation as resolved.
Show resolved Hide resolved
const id = setTimeout(() => controller.abort(), this.timeout);

// assign data
// assign database
if (data) {
data.dbName = data.dbName || this.database;
shanghaikid marked this conversation as resolved.
Show resolved Hide resolved
}

const response = await fetch(`${this.baseURL}${url}`, {
const response = await this.fetch(`${this.baseURL}${url}`, {
method: 'post',
headers: this.headers,
body: JSON.stringify(data),
Expand All @@ -78,41 +112,45 @@ export class HttpBaseClient {
return response.json() as T;
shanghaikid marked this conversation as resolved.
Show resolved Hide resolved
} catch (error) {
if (error.name === 'AbortError') {
console.log('request was timeout');
console.warn('milvus http client: request was timeout');
}
return Promise.reject(error);
}
}

// GET API
async GET<T>(url: string, params: Record<string, any> = {}): Promise<T> {
try {
const controller = new AbortController();
const id = setTimeout(() => controller.abort(), this.timeout);

// assign data
// assign database
if (params) {
params.dbName = params.dbName || this.database;
}

const queryParams = new URLSearchParams(params);

const response = await fetch(`${this.baseURL}${url}?${queryParams}`, {
method: 'get',
headers: this.headers,
signal: controller.signal,
});
const response = await this.fetch(
`${this.baseURL}${url}?${queryParams}`,
{
method: 'get',
headers: this.headers,
signal: controller.signal,
}
);

clearTimeout(id);

return response.json() as T;
} catch (error) {
if (error.name === 'AbortError') {
console.log('request was timeout');
console.warn('milvus http client: request was timeout');
}
return Promise.reject(error);
}
}
}

// mixin APIs
// The HttpClient class extends the functionality of the HttpBaseClient class by mixing in the Collection and Vector APIs.
export class HttpClient extends Collection(Vector(HttpBaseClient)) {}
12 changes: 12 additions & 0 deletions milvus/http/Collection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,18 @@ import {
DEFAULT_VECTOR_FIELD,
} from '../const';

/**
* Collection is a mixin function that extends the functionality of a base class.
* It provides methods to interact with collections in a Milvus cluster.
*
* @param {Constructor<HttpBaseClient>} Base - The base class to be extended.
* @returns {class} - The extended class with additional methods for collection management.
*
* @method createCollection - Creates a new collection in Milvus.
* @method describeCollection - Retrieves the description of a specific collection.
* @method dropCollection - Deletes a specific collection from Milvus.
* @method listCollections - Lists all collections in the Milvus cluster.
*/
export function Collection<T extends Constructor<HttpBaseClient>>(Base: T) {
return class extends Base {
// POST create collection
Expand Down
14 changes: 14 additions & 0 deletions milvus/http/Vector.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,20 @@ import {
HttpBaseResponse,
} from '../types';

/**
* Vector is a mixin function that extends the functionality of a base class.
* It provides methods to interact with vectors in a Milvus cluster.
*
* @param {Constructor<HttpBaseClient>} Base - The base class to be extended.
* @returns {class} - The extended class with additional methods for vector management.
*
* @method get - Retrieves a specific vector from Milvus.
* @method insert - Inserts a new vector into Milvus.
* @method upsert - Inserts a new vector into Milvus, or updates it if it already exists.
* @method query - Queries for vectors in Milvus.
* @method search - Searches for vectors in Milvus.
* @method delete - Deletes a specific vector from Milvus.
*/
export function Vector<T extends Constructor<HttpBaseClient>>(Base: T) {
return class extends Base {
// GET get data
Expand Down
4 changes: 4 additions & 0 deletions milvus/types/Http.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
import { FloatVectors } from '..';
type Fetch = (input: any, init?: any) => Promise<any>;

// Class types
export type Constructor<T = {}> = new (...args: any[]) => T;

Expand All @@ -15,6 +17,8 @@ type HttpClientConfigBase = {
password?: string;
// request timeout, number in milliseconds.
timeout?: number;
// altenative fetch api
fetch?: Fetch;
};

type HttpClientConfigAddress = HttpClientConfigBase & {
Expand Down
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@
"@grpc/proto-loader": "0.7.7",
"dayjs": "^1.11.7",
"lru-cache": "^9.1.2",
"node-fetch": "2",
"protobufjs": "7.2.4",
"winston": "^3.9.0"
},
"devDependencies": {
"@babel/plugin-transform-modules-commonjs": "^7.21.5",
"@types/jest": "^29.5.1",
"@types/node-fetch": "^2.6.7",
"@types/node-fetch": "^2.6.8",
"jest": "^29.5.0",
"node-fetch": "2",
"ts-jest": "^29.1.0",
"ts-node": "^10.9.1",
"typedoc": "^0.24.7",
Expand Down
16 changes: 16 additions & 0 deletions test/http/client.spec.ts
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { default as nodeFetch } from 'node-fetch';
import {
HttpClient,
DEFAULT_DB,
Expand Down Expand Up @@ -112,4 +113,19 @@ describe(`HTTP Client test`, () => {
const client2 = new HttpClient(config2);
expect(client2.timeout).toBe(timeout);
});

it('should using the correct fetch', () => {
const config = {
baseURL,
fetch: nodeFetch,
};
const client = new HttpClient(config);
expect(client.fetch).toEqual(nodeFetch);

const config2 = {
baseURL,
};
const client2 = new HttpClient(config2);
expect(client2.fetch).toEqual(fetch);
});
});
2 changes: 1 addition & 1 deletion test/http/cloud.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@ const config = {
password: 'password',
};

generateTests(config);
generateTests('cloud decidated api test', config);
2 changes: 1 addition & 1 deletion test/http/api.spec.ts → test/http/fetch.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@ import { generateTests } from './test';
const config = {
endpoint: ENDPOINT,
};
generateTests(config);
generateTests('http api by native fetch test', config);
9 changes: 9 additions & 0 deletions test/http/node-fetch.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import fetch from 'node-fetch';
import { ENDPOINT } from '../tools';
import { generateTests } from './test';

const config = {
endpoint: ENDPOINT,
fetch: fetch,
};
generateTests('http api by node-fetch v2 test', config);
2 changes: 1 addition & 1 deletion test/http/serverless.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ const config = {
token: 'serverless api key',
};

generateTests(config);
generateTests('serverless api test', config);
10 changes: 5 additions & 5 deletions test/http/test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,12 @@ import {
dynamicFields,
} from '../tools';

export function generateTests(config: HttpClientConfig) {
describe(`HTTP API tests`, () => {
export function generateTests(
desc = `HTTP API tests`,
config: HttpClientConfig
) {
describe(desc, () => {
// Mock configuration object
const dbParam = {
db_name: 'HttpClient_collections',
};
const createParams = {
dimension: 4,
collectionName: 'my_collection',
Expand Down
8 changes: 4 additions & 4 deletions yarn.lock
Original file line number Diff line number Diff line change
Expand Up @@ -879,10 +879,10 @@
resolved "https://registry.yarnpkg.com/@types/long/-/long-4.0.2.tgz#b74129719fc8d11c01868010082d483b7545591a"
integrity sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==

"@types/node-fetch@^2.6.7":
version "2.6.7"
resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.7.tgz#a1abe2ce24228b58ad97f99480fdcf9bbc6ab16d"
integrity sha512-lX17GZVpJ/fuCjguZ5b3TjEbSENxmEk1B2z02yoXSK9WMEWRivhdSY73wWMn6bpcCDAOh6qAdktpKHIlkDk2lg==
"@types/node-fetch@^2.6.8":
version "2.6.8"
resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.8.tgz#9a2993583975849c2e1f360b6ca2f11755b2c504"
integrity sha512-nnH5lV9QCMPsbEVdTb5Y+F3GQxLSw1xQgIydrb2gSfEavRPs50FnMr+KUaa+LoPSqibm2N+ZZxH7lavZlAT4GA==
dependencies:
"@types/node" "*"
form-data "^4.0.0"
Expand Down