From c6e0b02dd7801ed55f2c478924789bf84c006ec4 Mon Sep 17 00:00:00 2001 From: Alex Freska Date: Thu, 18 Apr 2024 10:50:24 -0400 Subject: [PATCH] fix: renterd-js and core request api --- .changeset/shy-fans-know.md | 5 ++ libs/renterd-js/README.md | 5 -- libs/renterd-js/src/autopilot.ts | 1 + libs/renterd-js/src/bus.ts | 1 + libs/renterd-js/src/example.ts | 5 -- libs/renterd-js/src/index.spec.ts | 80 ++++++++++++++++++++++++++++ libs/renterd-js/src/worker.ts | 23 +++++++-- libs/request/package.json | 3 +- libs/request/src/index.ts | 54 ++++++++++--------- libs/sia-central-js/src/index.ts | 18 ++++--- package-lock.json | 86 +++++++++++++++---------------- package.json | 4 +- 12 files changed, 195 insertions(+), 90 deletions(-) create mode 100644 .changeset/shy-fans-know.md create mode 100644 libs/renterd-js/src/index.spec.ts diff --git a/.changeset/shy-fans-know.md b/.changeset/shy-fans-know.md new file mode 100644 index 000000000..c24636fd2 --- /dev/null +++ b/.changeset/shy-fans-know.md @@ -0,0 +1,5 @@ +--- +'@siafoundation/renterd-js': patch +--- + +Fixed an issue with upload and download content type and encoding. Closes https://github.com/SiaFoundation/web/issues/591 diff --git a/libs/renterd-js/README.md b/libs/renterd-js/README.md index aa314ce38..210d8f898 100644 --- a/libs/renterd-js/README.md +++ b/libs/renterd-js/README.md @@ -84,11 +84,6 @@ export async function example() { key: 'path/to/file.txt', bucket: 'my-bucket', }, - config: { - onDownloadProgress: (progress) => { - console.log(progress.loaded / progress.total) - }, - }, }) } ``` diff --git a/libs/renterd-js/src/autopilot.ts b/libs/renterd-js/src/autopilot.ts index 321c6f0e4..40e063482 100644 --- a/libs/renterd-js/src/autopilot.ts +++ b/libs/renterd-js/src/autopilot.ts @@ -30,6 +30,7 @@ export function Autopilot({ }) { const axios = initAxios(api, password) return { + axios, state: buildRequestHandler< AutopilotStateParams, AutopilotStatePayload, diff --git a/libs/renterd-js/src/bus.ts b/libs/renterd-js/src/bus.ts index ae1b150c0..659b2088f 100644 --- a/libs/renterd-js/src/bus.ts +++ b/libs/renterd-js/src/bus.ts @@ -268,6 +268,7 @@ import { buildRequestHandler, initAxios } from '@siafoundation/request' export function Bus({ api, password }: { api: string; password?: string }) { const axios = initAxios(api, password) return { + axios, busState: buildRequestHandler< BusStateParams, BusStatePayload, diff --git a/libs/renterd-js/src/example.ts b/libs/renterd-js/src/example.ts index e75a038fa..87cd25d32 100644 --- a/libs/renterd-js/src/example.ts +++ b/libs/renterd-js/src/example.ts @@ -73,10 +73,5 @@ export async function example() { key: 'path/to/file.txt', bucket: 'my-bucket', }, - config: { - onDownloadProgress: (progress) => { - console.log(progress.loaded / progress.total) - }, - }, }) } diff --git a/libs/renterd-js/src/index.spec.ts b/libs/renterd-js/src/index.spec.ts new file mode 100644 index 000000000..4e8488a75 --- /dev/null +++ b/libs/renterd-js/src/index.spec.ts @@ -0,0 +1,80 @@ +import MockAdapter from 'axios-mock-adapter' +import { Bus } from './bus' +import { Worker } from './worker' + +describe('renterd-js', () => { + const api = 'https://sia.tech/api' + const password = 'password1337' + + it('default data and headers support json', async () => { + const bus = Bus({ + api, + password, + }) + const mock = new MockAdapter(bus.axios) + + const bucket = 'newbucket' + const url = `/bus/buckets` + const fullUrl = `${api}${url}` + + mock.onPost(fullUrl).reply((config) => { + expect(config.url).toBe(url) + expect(config.method).toBe('post') + expect(config.headers?.['Content-Type']).toMatch(/application\/json/) + expect(config.data).toEqual( + JSON.stringify({ + name: bucket, + }) + ) + return [200, { name: bucket }] + }) + + const response = await bus.bucketCreate({ + data: { + name: bucket, + }, + }) + + expect(response.status).toBe(200) + }) + + it('object upload should send file as multipart form data', async () => { + const worker = Worker({ + api, + password, + }) + const mock = new MockAdapter(worker.axios) + + const bucket = 'default' + const name = 'test.txt' + const fileKey = `directory/${name}` + const url = `/worker/objects/${fileKey}?bucket=${bucket}` + const fullUrl = `${api}${url}` + + mock.onPut(fullUrl).reply((config) => { + expect(config.url).toBe(url) + expect(config.method).toBe('put') + expect(config.headers?.['Content-Type']).toMatch(/multipart\/form-data/) + expect(config.data).toBeInstanceOf(File) + if (config.onUploadProgress) { + config.onUploadProgress({ loaded: 500, total: 1000 }) + } + return [200, undefined] + }) + + const response = await worker.objectUpload({ + params: { + key: fileKey, + bucket: bucket, + }, + data: new File(['hello world'], name), + config: { + onUploadProgress: (progress) => { + expect(progress.loaded / progress.total).toBe(0.5) + }, + }, + }) + + expect(response.status).toBe(200) + }) +}) diff --git a/libs/renterd-js/src/worker.ts b/libs/renterd-js/src/worker.ts index 57a0ef4a1..23a6d1713 100644 --- a/libs/renterd-js/src/worker.ts +++ b/libs/renterd-js/src/worker.ts @@ -24,6 +24,7 @@ import { buildRequestHandler, initAxios } from '@siafoundation/request' export function Worker({ api, password }: { api: string; password?: string }) { const axios = initAxios(api, password) return { + axios, workerState: buildRequestHandler< WorkerStateParams, WorkerStatePayload, @@ -33,17 +34,33 @@ export function Worker({ api, password }: { api: string; password?: string }) { ObjectDownloadParams, ObjectDownloadPayload, ObjectDownloadResponse - >(axios, 'get', workerObjectsKeyRoute), + >(axios, 'get', workerObjectsKeyRoute, { + config: { + responseType: 'blob', + }, + }), objectUpload: buildRequestHandler< ObjectUploadParams, ObjectUploadPayload, ObjectUploadResponse - >(axios, 'put', workerObjectsKeyRoute), + >(axios, 'put', workerObjectsKeyRoute, { + config: { + headers: { + 'Content-Type': 'multipart/form-data', + }, + }, + }), multipartUploadPart: buildRequestHandler< MultipartUploadPartParams, MultipartUploadPartPayload, MultipartUploadPartResponse - >(axios, 'put', workerMultipartKeyRoute), + >(axios, 'put', workerMultipartKeyRoute, { + config: { + headers: { + 'Content-Type': 'multipart/form-data', + }, + }, + }), rhpScan: buildRequestHandler< RhpScanParams, RhpScanPayload, diff --git a/libs/request/package.json b/libs/request/package.json index f70ac0036..a4b2c5328 100644 --- a/libs/request/package.json +++ b/libs/request/package.json @@ -4,7 +4,8 @@ "version": "0.1.0", "license": "MIT", "dependencies": { - "axios": "^0.27.2" + "axios": "^0.27.2", + "@technically/lodash": "^4.17.0" }, "types": "./src/index.d.ts" } diff --git a/libs/request/src/index.ts b/libs/request/src/index.ts index 2c13ca370..3cea78132 100644 --- a/libs/request/src/index.ts +++ b/libs/request/src/index.ts @@ -1,6 +1,7 @@ -import { - Axios, +import { merge } from '@technically/lodash' +import axios, { AxiosError, + AxiosInstance, AxiosRequestConfig, AxiosRequestHeaders, AxiosResponse, @@ -39,7 +40,15 @@ export function buildRequestHandler< Params = void, Data = void, Response = void ->(axios: Axios, method: Method, route: string, defaultParams?: Params) { +>( + axios: AxiosInstance, + method: Method, + route: string, + options: { + defaultParams?: Params + config?: AxiosRequestConfig + } = {} +) { type Args = Params extends void ? Data extends void ? { config?: AxiosRequestConfig } | void @@ -55,26 +64,32 @@ export function buildRequestHandler< config?: AxiosRequestConfig } + type MaybeArgs = { + params?: Params + data?: Data + config?: AxiosRequestConfig + } + return (args: Args) => { - // args is sometimes undefined - const a = { + // Force remove the void type + const nonVoidArgs: MaybeArgs = { ...args, - } as { - params?: Params - data?: Data - config?: AxiosRequestConfig + } + const mergedArgs: MaybeArgs = { + ...nonVoidArgs, + config: merge(options.config, nonVoidArgs.config), } const params = { - ...defaultParams, - ...a.params, + ...options.defaultParams, + ...mergedArgs.params, } // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const paramRoute = parameterizeRoute(route, params as RequestParams)! - const data = 'data' in a ? JSON.stringify(a.data) : undefined + const data = 'data' in mergedArgs ? mergedArgs.data : undefined - return axios[method](paramRoute, data, a?.config) + return axios[method](paramRoute, data, mergedArgs?.config) } } @@ -89,28 +104,21 @@ export async function to( > { try { const response = await promise - // Just in case the response is not already JSON - try { - const data = JSON.parse(response.data as string) - return [data, undefined, response] - } catch (e) { - return [response.data, undefined, response] - } + return [response.data, undefined, response] } catch (error) { return [undefined, error as AxiosError, undefined] } } -export function initAxios(api: string, password?: string): Axios { +export function initAxios(api: string, password?: string): AxiosInstance { const headers: AxiosRequestHeaders = { 'Content-Type': 'application/json', } if (password) { headers['Authorization'] = `Basic ${btoa(`:${password}`)}` } - return new Axios({ + return axios.create({ baseURL: api, headers, - responseType: 'json', }) } diff --git a/libs/sia-central-js/src/index.ts b/libs/sia-central-js/src/index.ts index 3d393a0e9..3a2efc293 100644 --- a/libs/sia-central-js/src/index.ts +++ b/libs/sia-central-js/src/index.ts @@ -72,7 +72,9 @@ export function SiaCentral({ api }: { api: string } = { api: defaultApi }) { SiaCentralExchangeRatesPayload, SiaCentralExchangeRatesResponse >(axios, 'get', '/market/exchange-rate', { - currencies: 'sc', + defaultParams: { + currencies: 'sc', + }, }), host: buildRequestHandler< SiaCentralHostParams, @@ -84,12 +86,14 @@ export function SiaCentral({ api }: { api: string } = { api: defaultApi }) { SiaCentralHostsPayload, SiaCentralHostsResponse >(axios, 'get', '/hosts/list', { - showinactive: false, - sort: 'used_storage', - dir: 'desc', - protocol: 'rhp3', - limit: 10, - page: 1, + defaultParams: { + showinactive: false, + sort: 'used_storage', + dir: 'desc', + protocol: 'rhp3', + limit: 10, + page: 1, + }, }), hostsNetworkAverages: buildRequestHandler< SiaCentralHostsNetworkAveragesParams, diff --git a/package-lock.json b/package-lock.json index f2b8704f3..d71f5373e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -78,13 +78,11 @@ "html-to-image": "^1.11.11", "identicon.js": "^2.3.3", "jest-environment-jsdom": "29.4.3", - "lowdb": "^3.0.0", "next": "14.0.4", "next-mdx-remote": "^4.0.3", "next-themes": "^0.2.1", "node-cron": "^3.0.2", "node-fetch": "^3.3.2", - "playwright": "^1.42.1", "react": "18.2.0", "react-dom": "18.2.0", "react-dropzone": "^14.2.3", @@ -153,6 +151,7 @@ "@typescript-eslint/eslint-plugin": "6.21.0", "@typescript-eslint/parser": "6.21.0", "autoprefixer": "10.4.13", + "axios-mock-adapter": "^1.22.0", "babel-jest": "29.4.3", "css-loader": "^6.4.0", "eslint": "8.48.0", @@ -167,6 +166,7 @@ "jest": "29.4.3", "msw": "^2.1.7", "nx": "18.0.3", + "playwright": "^1.42.1", "postcss": "8.4.21", "prettier": "2.7.1", "react-refresh": "^0.10.0", @@ -365,6 +365,7 @@ "extraneous": true }, "libs/hostd-react": { + "name": "@siafoundation/hostd-react", "version": "4.1.0", "license": "MIT", "dependencies": { @@ -374,6 +375,7 @@ } }, "libs/hostd-types": { + "name": "@siafoundation/hostd-types", "version": "0.1.0", "license": "MIT", "dependencies": { @@ -507,6 +509,7 @@ } }, "libs/renterd-js": { + "name": "@siafoundation/renterd-js", "version": "0.1.0", "license": "MIT", "dependencies": { @@ -515,6 +518,7 @@ } }, "libs/renterd-react": { + "name": "@siafoundation/renterd-react", "version": "4.1.0", "license": "MIT", "dependencies": { @@ -525,6 +529,7 @@ } }, "libs/renterd-types": { + "name": "@siafoundation/renterd-types", "version": "0.1.0", "license": "MIT", "dependencies": { @@ -532,6 +537,7 @@ } }, "libs/request": { + "name": "@siafoundation/request", "version": "0.1.0", "license": "MIT", "dependencies": { @@ -563,14 +569,16 @@ } }, "libs/sia-central-js": { + "name": "@siafoundation/sia-central-js", "version": "0.4.0", "license": "MIT", "dependencies": { - "@siafoundation/sia-central-types": "0.1.0", - "@technically/lodash": "^4.17.0" + "@siafoundation/request": "0.1.0", + "@siafoundation/sia-central-types": "0.1.0" } }, "libs/sia-central-mock": { + "name": "@siafoundation/sia-central-mock", "version": "0.1.0", "license": "MIT", "dependencies": { @@ -579,6 +587,7 @@ } }, "libs/sia-central-react": { + "name": "@siafoundation/sia-central-react", "version": "3.1.0", "license": "MIT", "dependencies": { @@ -587,6 +596,7 @@ } }, "libs/sia-central-types": { + "name": "@siafoundation/sia-central-types", "version": "0.1.0", "license": "MIT" }, @@ -638,6 +648,7 @@ } }, "libs/walletd-mock": { + "name": "@siafoundation/walletd-mock", "version": "0.1.0", "license": "MIT", "dependencies": { @@ -649,6 +660,7 @@ } }, "libs/walletd-react": { + "name": "@siafoundation/walletd-react", "version": "4.1.0", "license": "MIT", "dependencies": { @@ -658,6 +670,7 @@ } }, "libs/walletd-types": { + "name": "@siafoundation/walletd-types", "version": "0.1.0", "license": "MIT", "dependencies": { @@ -10668,6 +10681,19 @@ "form-data": "^4.0.0" } }, + "node_modules/axios-mock-adapter": { + "version": "1.22.0", + "resolved": "https://registry.npmjs.org/axios-mock-adapter/-/axios-mock-adapter-1.22.0.tgz", + "integrity": "sha512-dmI0KbkyAhntUR05YY96qg2H6gg0XMl2+qTW0xmYg6Up+BFBAJYRLROMXRdDEL06/Wqwa0TJThAYvFtSFdRCZw==", + "dev": true, + "dependencies": { + "fast-deep-equal": "^3.1.3", + "is-buffer": "^2.0.5" + }, + "peerDependencies": { + "axios": ">= 0.17.0" + } + }, "node_modules/axobject-query": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-3.2.1.tgz", @@ -19291,20 +19317,6 @@ "loose-envify": "cli.js" } }, - "node_modules/lowdb": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/lowdb/-/lowdb-3.0.0.tgz", - "integrity": "sha512-9KZRulmIcU8fZuWiaM0d5e2/nPnrFyXkeXVpqT+MJS+vgbgOf1EbtvgQmba8HwUFgDl1oeZR6XqEJnkJmQdKmg==", - "dependencies": { - "steno": "^2.1.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/typicode" - } - }, "node_modules/lower-case": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", @@ -25624,17 +25636,6 @@ "node": ">=0.10.0" } }, - "node_modules/steno": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/steno/-/steno-2.1.0.tgz", - "integrity": "sha512-mauOsiaqTNGFkWqIfwcm3y/fq+qKKaIWf1vf3ocOuTdco9XoHCO2AGF1gFYXuZFSWuP38Q8LBHBGJv2KnJSXyA==", - "engines": { - "node": "^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/typicode" - } - }, "node_modules/stop-iteration-iterator": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz", @@ -33893,8 +33894,8 @@ "@siafoundation/sia-central-js": { "version": "file:libs/sia-central-js", "requires": { - "@siafoundation/sia-central-types": "0.1.0", - "@technically/lodash": "^4.17.0" + "@siafoundation/request": "0.1.0", + "@siafoundation/sia-central-types": "0.1.0" } }, "@siafoundation/sia-central-mock": { @@ -36175,6 +36176,16 @@ "form-data": "^4.0.0" } }, + "axios-mock-adapter": { + "version": "1.22.0", + "resolved": "https://registry.npmjs.org/axios-mock-adapter/-/axios-mock-adapter-1.22.0.tgz", + "integrity": "sha512-dmI0KbkyAhntUR05YY96qg2H6gg0XMl2+qTW0xmYg6Up+BFBAJYRLROMXRdDEL06/Wqwa0TJThAYvFtSFdRCZw==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.3", + "is-buffer": "^2.0.5" + } + }, "axobject-query": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-3.2.1.tgz", @@ -42459,14 +42470,6 @@ "js-tokens": "^3.0.0 || ^4.0.0" } }, - "lowdb": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/lowdb/-/lowdb-3.0.0.tgz", - "integrity": "sha512-9KZRulmIcU8fZuWiaM0d5e2/nPnrFyXkeXVpqT+MJS+vgbgOf1EbtvgQmba8HwUFgDl1oeZR6XqEJnkJmQdKmg==", - "requires": { - "steno": "^2.1.0" - } - }, "lower-case": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/lower-case/-/lower-case-2.0.2.tgz", @@ -46929,11 +46932,6 @@ "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz", "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=" }, - "steno": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/steno/-/steno-2.1.0.tgz", - "integrity": "sha512-mauOsiaqTNGFkWqIfwcm3y/fq+qKKaIWf1vf3ocOuTdco9XoHCO2AGF1gFYXuZFSWuP38Q8LBHBGJv2KnJSXyA==" - }, "stop-iteration-iterator": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/stop-iteration-iterator/-/stop-iteration-iterator-1.0.0.tgz", diff --git a/package.json b/package.json index 52c921dba..4e3d863bd 100644 --- a/package.json +++ b/package.json @@ -90,13 +90,11 @@ "html-to-image": "^1.11.11", "identicon.js": "^2.3.3", "jest-environment-jsdom": "29.4.3", - "lowdb": "^3.0.0", "next": "14.0.4", "next-mdx-remote": "^4.0.3", "next-themes": "^0.2.1", "node-cron": "^3.0.2", "node-fetch": "^3.3.2", - "playwright": "^1.42.1", "react": "18.2.0", "react-dom": "18.2.0", "react-dropzone": "^14.2.3", @@ -165,6 +163,8 @@ "@typescript-eslint/eslint-plugin": "6.21.0", "@typescript-eslint/parser": "6.21.0", "autoprefixer": "10.4.13", + "axios-mock-adapter": "^1.22.0", + "playwright": "^1.42.1", "babel-jest": "29.4.3", "css-loader": "^6.4.0", "eslint": "8.48.0",