From dc0295c31455ed37fa6902cf04526de03b3f5767 Mon Sep 17 00:00:00 2001 From: Henry Date: Sat, 11 Jan 2025 15:14:29 +0000 Subject: [PATCH 1/2] add s3 to multer --- packages/components/src/storageUtils.ts | 103 ++++++++++++++++++ packages/server/package.json | 2 + .../openai-assistants-vector-store/index.ts | 2 +- .../controllers/openai-assistants/index.ts | 2 +- .../server/src/routes/attachments/index.ts | 7 +- .../server/src/routes/documentstore/index.ts | 6 +- .../routes/openai-assistants-files/index.ts | 6 +- .../openai-assistants-vector-store/index.ts | 6 +- .../server/src/routes/predictions/index.ts | 12 +- packages/server/src/routes/vectors/index.ts | 9 +- .../src/services/documentstore/index.ts | 9 +- .../openai-assistants-vector-store/index.ts | 7 +- .../src/services/openai-assistants/index.ts | 7 +- packages/server/src/utils/buildChatflow.ts | 9 +- packages/server/src/utils/createAttachment.ts | 14 ++- packages/server/src/utils/index.ts | 21 ---- packages/server/src/utils/telemetry.ts | 3 +- packages/server/src/utils/upsertVector.ts | 15 ++- pnpm-lock.yaml | 93 ++++++++++++++-- 19 files changed, 250 insertions(+), 83 deletions(-) diff --git a/packages/components/src/storageUtils.ts b/packages/components/src/storageUtils.ts index 2b7146cde02..6b365f0d9eb 100644 --- a/packages/components/src/storageUtils.ts +++ b/packages/components/src/storageUtils.ts @@ -11,6 +11,64 @@ import { import { Readable } from 'node:stream' import { getUserHome } from './utils' import sanitize from 'sanitize-filename' +import multer from 'multer' +import multerS3 from 'multer-s3' + +/** + * Get user settings file + * TODO: move env variables to settings json file, easier configuration + */ +export const getUserSettingsFilePath = () => { + if (process.env.SECRETKEY_PATH) return path.join(process.env.SECRETKEY_PATH, 'settings.json') + const checkPaths = [path.join(getUserHome(), '.flowise', 'settings.json')] + for (const checkPath of checkPaths) { + if (fs.existsSync(checkPath)) { + return checkPath + } + } + return '' +} + +export const getOrgId = () => { + const settingsContent = fs.readFileSync(getUserSettingsFilePath(), 'utf8') + try { + const settings = JSON.parse(settingsContent) + return settings.instanceId + } catch (error) { + return '' + } +} + +const getUploadPath = (): string => { + return process.env.BLOB_STORAGE_PATH + ? path.join(process.env.BLOB_STORAGE_PATH, 'uploads', getOrgId()) + : path.join(getUserHome(), '.flowise', 'uploads', getOrgId()) +} + +export const getMulterStorage = () => { + const storageType = getStorageType() + + if (storageType === 's3') { + const s3Client = getS3Config().s3Client + const Bucket = getS3Config().Bucket + + const upload = multer({ + storage: multerS3({ + s3: s3Client, + bucket: Bucket, + metadata: function (req, file, cb) { + cb(null, { fieldName: file.fieldname, originalName: file.originalname, orgId: getOrgId() }) + }, + key: function (req, file, cb) { + cb(null, `${getOrgId()}/${Date.now().toString()}`) + } + }) + }) + return upload + } else { + return multer({ dest: getUploadPath() }) + } +} export const addBase64FilesToStorage = async (fileBase64: string, chatflowid: string, fileNames: string[]) => { const storageType = getStorageType() @@ -120,6 +178,37 @@ export const addSingleFileToStorage = async (mime: string, bf: Buffer, fileName: } } +export const getFileFromUpload = async (filePath: string): Promise => { + const storageType = getStorageType() + if (storageType === 's3') { + const { s3Client, Bucket } = getS3Config() + + let Key = filePath + // remove the first '/' if it exists + if (Key.startsWith('/')) { + Key = Key.substring(1) + } + const getParams = { + Bucket, + Key + } + + const response = await s3Client.send(new GetObjectCommand(getParams)) + const body = response.Body + if (body instanceof Readable) { + const streamToString = await body.transformToString('base64') + if (streamToString) { + return Buffer.from(streamToString, 'base64') + } + } + // @ts-ignore + const buffer = Buffer.concat(response.Body.toArray()) + return buffer + } else { + return fs.readFileSync(filePath) + } +} + export const getFileFromStorage = async (file: string, ...paths: string[]): Promise => { const storageType = getStorageType() const sanitizedFilename = _sanitizeFilename(file) @@ -183,6 +272,20 @@ export const removeFilesFromStorage = async (...paths: string[]) => { } } +export const removeSpecificFileFromUpload = async (filePath: string) => { + const storageType = getStorageType() + if (storageType === 's3') { + let Key = filePath + // remove the first '/' if it exists + if (Key.startsWith('/')) { + Key = Key.substring(1) + } + await _deleteS3Folder(Key) + } else { + fs.unlinkSync(filePath) + } +} + export const removeSpecificFileFromStorage = async (...paths: string[]) => { const storageType = getStorageType() if (storageType === 's3') { diff --git a/packages/server/package.json b/packages/server/package.json index 34dc81844f6..6b2f5b7b525 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -91,6 +91,7 @@ "moment": "^2.29.3", "moment-timezone": "^0.5.34", "multer": "^1.4.5-lts.1", + "multer-s3": "^3.0.1", "mysql2": "^3.11.3", "openai": "^4.57.3", "pg": "^8.11.1", @@ -109,6 +110,7 @@ "@types/cors": "^2.8.12", "@types/crypto-js": "^4.1.1", "@types/multer": "^1.4.7", + "@types/multer-s3": "^3.0.3", "@types/sanitize-html": "^2.9.5", "concurrently": "^7.1.0", "cypress": "^13.13.0", diff --git a/packages/server/src/controllers/openai-assistants-vector-store/index.ts b/packages/server/src/controllers/openai-assistants-vector-store/index.ts index 2b82ca0d078..7f8b430b4ea 100644 --- a/packages/server/src/controllers/openai-assistants-vector-store/index.ts +++ b/packages/server/src/controllers/openai-assistants-vector-store/index.ts @@ -143,7 +143,7 @@ const uploadFilesToAssistantVectorStore = async (req: Request, res: Response, ne // Address file name with special characters: https://github.com/expressjs/multer/issues/1104 file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8') uploadFiles.push({ - filePath: file.path, + filePath: file.path ?? (file as any).key, fileName: file.originalname }) } diff --git a/packages/server/src/controllers/openai-assistants/index.ts b/packages/server/src/controllers/openai-assistants/index.ts index 94c2afe69e7..dc660199dee 100644 --- a/packages/server/src/controllers/openai-assistants/index.ts +++ b/packages/server/src/controllers/openai-assistants/index.ts @@ -84,7 +84,7 @@ const uploadAssistantFiles = async (req: Request, res: Response, next: NextFunct // Address file name with special characters: https://github.com/expressjs/multer/issues/1104 file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8') uploadFiles.push({ - filePath: file.path, + filePath: file.path ?? (file as any).key, fileName: file.originalname }) } diff --git a/packages/server/src/routes/attachments/index.ts b/packages/server/src/routes/attachments/index.ts index a4223781f72..99d07714c86 100644 --- a/packages/server/src/routes/attachments/index.ts +++ b/packages/server/src/routes/attachments/index.ts @@ -1,13 +1,10 @@ import express from 'express' -import multer from 'multer' import attachmentsController from '../../controllers/attachments' -import { getUploadPath } from '../../utils' +import { getMulterStorage } from 'flowise-components' const router = express.Router() -const upload = multer({ dest: getUploadPath() }) - // CREATE -router.post('/:chatflowId/:chatId', upload.array('files'), attachmentsController.createAttachment) +router.post('/:chatflowId/:chatId', getMulterStorage().array('files'), attachmentsController.createAttachment) export default router diff --git a/packages/server/src/routes/documentstore/index.ts b/packages/server/src/routes/documentstore/index.ts index b53ceb3c5cc..3e7e9b4f42d 100644 --- a/packages/server/src/routes/documentstore/index.ts +++ b/packages/server/src/routes/documentstore/index.ts @@ -1,12 +1,10 @@ import express from 'express' -import multer from 'multer' -import { getUploadPath } from '../../utils' import documentStoreController from '../../controllers/documentstore' +import { getMulterStorage } from 'flowise-components' const router = express.Router() -const upload = multer({ dest: getUploadPath() }) -router.post(['/upsert/', '/upsert/:id'], upload.array('files'), documentStoreController.upsertDocStoreMiddleware) +router.post(['/upsert/', '/upsert/:id'], getMulterStorage().array('files'), documentStoreController.upsertDocStoreMiddleware) router.post(['/refresh/', '/refresh/:id'], documentStoreController.refreshDocStoreMiddleware) diff --git a/packages/server/src/routes/openai-assistants-files/index.ts b/packages/server/src/routes/openai-assistants-files/index.ts index d717754715f..40ab30fce6d 100644 --- a/packages/server/src/routes/openai-assistants-files/index.ts +++ b/packages/server/src/routes/openai-assistants-files/index.ts @@ -1,12 +1,10 @@ import express from 'express' -import multer from 'multer' import openaiAssistantsController from '../../controllers/openai-assistants' -import { getUploadPath } from '../../utils' +import { getMulterStorage } from 'flowise-components' const router = express.Router() -const upload = multer({ dest: getUploadPath() }) router.post('/download/', openaiAssistantsController.getFileFromAssistant) -router.post('/upload/', upload.array('files'), openaiAssistantsController.uploadAssistantFiles) +router.post('/upload/', getMulterStorage().array('files'), openaiAssistantsController.uploadAssistantFiles) export default router diff --git a/packages/server/src/routes/openai-assistants-vector-store/index.ts b/packages/server/src/routes/openai-assistants-vector-store/index.ts index a6721566c12..0db457befb4 100644 --- a/packages/server/src/routes/openai-assistants-vector-store/index.ts +++ b/packages/server/src/routes/openai-assistants-vector-store/index.ts @@ -1,10 +1,8 @@ import express from 'express' -import multer from 'multer' import openaiAssistantsVectorStoreController from '../../controllers/openai-assistants-vector-store' -import { getUploadPath } from '../../utils' +import { getMulterStorage } from 'flowise-components' const router = express.Router() -const upload = multer({ dest: getUploadPath() }) // CREATE router.post('/', openaiAssistantsVectorStoreController.createAssistantVectorStore) @@ -22,7 +20,7 @@ router.put(['/', '/:id'], openaiAssistantsVectorStoreController.updateAssistantV router.delete(['/', '/:id'], openaiAssistantsVectorStoreController.deleteAssistantVectorStore) // POST -router.post('/:id', upload.array('files'), openaiAssistantsVectorStoreController.uploadFilesToAssistantVectorStore) +router.post('/:id', getMulterStorage().array('files'), openaiAssistantsVectorStoreController.uploadFilesToAssistantVectorStore) // DELETE router.patch(['/', '/:id'], openaiAssistantsVectorStoreController.deleteFilesFromAssistantVectorStore) diff --git a/packages/server/src/routes/predictions/index.ts b/packages/server/src/routes/predictions/index.ts index ca192c89fb2..e92843307c2 100644 --- a/packages/server/src/routes/predictions/index.ts +++ b/packages/server/src/routes/predictions/index.ts @@ -1,13 +1,15 @@ import express from 'express' -import multer from 'multer' import predictionsController from '../../controllers/predictions' -import { getUploadPath } from '../../utils' +import { getMulterStorage } from 'flowise-components' const router = express.Router() -const upload = multer({ dest: getUploadPath() }) - // CREATE -router.post(['/', '/:id'], upload.array('files'), predictionsController.getRateLimiterMiddleware, predictionsController.createPrediction) +router.post( + ['/', '/:id'], + getMulterStorage().array('files'), + predictionsController.getRateLimiterMiddleware, + predictionsController.createPrediction +) export default router diff --git a/packages/server/src/routes/vectors/index.ts b/packages/server/src/routes/vectors/index.ts index 060d7e4b34e..fa66fc27a57 100644 --- a/packages/server/src/routes/vectors/index.ts +++ b/packages/server/src/routes/vectors/index.ts @@ -1,19 +1,16 @@ import express from 'express' -import multer from 'multer' import vectorsController from '../../controllers/vectors' -import { getUploadPath } from '../../utils' +import { getMulterStorage } from 'flowise-components' const router = express.Router() -const upload = multer({ dest: getUploadPath() }) - // CREATE router.post( ['/upsert/', '/upsert/:id'], - upload.array('files'), + getMulterStorage().array('files'), vectorsController.getRateLimiterMiddleware, vectorsController.upsertVectorMiddleware ) -router.post(['/internal-upsert/', '/internal-upsert/:id'], upload.array('files'), vectorsController.createInternalUpsert) +router.post(['/internal-upsert/', '/internal-upsert/:id'], getMulterStorage().array('files'), vectorsController.createInternalUpsert) export default router diff --git a/packages/server/src/services/documentstore/index.ts b/packages/server/src/services/documentstore/index.ts index db2ecf322a2..564cf66dddd 100644 --- a/packages/server/src/services/documentstore/index.ts +++ b/packages/server/src/services/documentstore/index.ts @@ -1,17 +1,18 @@ import { getRunningExpressApp } from '../../utils/getRunningExpressApp' import { DocumentStore } from '../../database/entities/DocumentStore' -import * as fs from 'fs' import * as path from 'path' import { addArrayFilesToStorage, addSingleFileToStorage, getFileFromStorage, + getFileFromUpload, ICommonObject, IDocument, mapExtToInputField, mapMimeTypeToInputField, removeFilesFromStorage, - removeSpecificFileFromStorage + removeSpecificFileFromStorage, + removeSpecificFileFromUpload } from 'flowise-components' import { addLoaderSource, @@ -1441,7 +1442,7 @@ const upsertDocStoreMiddleware = async ( const filesLoaderConfig: ICommonObject = {} for (const file of files) { const fileNames: string[] = [] - const fileBuffer = fs.readFileSync(file.path) + const fileBuffer = await getFileFromUpload(file.path ?? (file as any).key) // Address file name with special characters: https://github.com/expressjs/multer/issues/1104 file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8') @@ -1481,7 +1482,7 @@ const upsertDocStoreMiddleware = async ( filesLoaderConfig[fileInputField] = JSON.stringify([storagePath]) } - fs.unlinkSync(file.path) + await removeSpecificFileFromUpload(file.path ?? (file as any).key) } loaderConfig = { diff --git a/packages/server/src/services/openai-assistants-vector-store/index.ts b/packages/server/src/services/openai-assistants-vector-store/index.ts index 46f9c183f8e..671e18d9098 100644 --- a/packages/server/src/services/openai-assistants-vector-store/index.ts +++ b/packages/server/src/services/openai-assistants-vector-store/index.ts @@ -1,11 +1,11 @@ import OpenAI from 'openai' import { StatusCodes } from 'http-status-codes' -import fs from 'fs' import { Credential } from '../../database/entities/Credential' import { InternalFlowiseError } from '../../errors/internalFlowiseError' import { getErrorMessage } from '../../errors/utils' import { getRunningExpressApp } from '../../utils/getRunningExpressApp' import { decryptCredentialData } from '../../utils' +import { getFileFromUpload, removeSpecificFileFromUpload } from 'flowise-components' const getAssistantVectorStore = async (credentialId: string, vectorStoreId: string) => { try { @@ -178,13 +178,14 @@ const uploadFilesToAssistantVectorStore = async ( const openai = new OpenAI({ apiKey: openAIApiKey }) const uploadedFiles = [] for (const file of files) { - const toFile = await OpenAI.toFile(fs.readFileSync(file.filePath), file.fileName) + const fileBuffer = await getFileFromUpload(file.filePath) + const toFile = await OpenAI.toFile(fileBuffer, file.fileName) const createdFile = await openai.files.create({ file: toFile, purpose: 'assistants' }) uploadedFiles.push(createdFile) - fs.unlinkSync(file.filePath) + await removeSpecificFileFromUpload(file.filePath) } const file_ids = [...uploadedFiles.map((file) => file.id)] diff --git a/packages/server/src/services/openai-assistants/index.ts b/packages/server/src/services/openai-assistants/index.ts index c908a546755..e842b04565b 100644 --- a/packages/server/src/services/openai-assistants/index.ts +++ b/packages/server/src/services/openai-assistants/index.ts @@ -1,11 +1,11 @@ import OpenAI from 'openai' -import fs from 'fs' import { StatusCodes } from 'http-status-codes' import { decryptCredentialData } from '../../utils' import { getRunningExpressApp } from '../../utils/getRunningExpressApp' import { Credential } from '../../database/entities/Credential' import { InternalFlowiseError } from '../../errors/internalFlowiseError' import { getErrorMessage } from '../../errors/utils' +import { getFileFromUpload, removeSpecificFileFromUpload } from 'flowise-components' // ---------------------------------------- // Assistants @@ -101,13 +101,14 @@ const uploadFilesToAssistant = async (credentialId: string, files: { filePath: s const uploadedFiles = [] for (const file of files) { - const toFile = await OpenAI.toFile(fs.readFileSync(file.filePath), file.fileName) + const fileBuffer = await getFileFromUpload(file.filePath) + const toFile = await OpenAI.toFile(fileBuffer, file.fileName) const createdFile = await openai.files.create({ file: toFile, purpose: 'assistants' }) uploadedFiles.push(createdFile) - fs.unlinkSync(file.filePath) + await removeSpecificFileFromUpload(file.filePath) } return uploadedFiles diff --git a/packages/server/src/utils/buildChatflow.ts b/packages/server/src/utils/buildChatflow.ts index 09cc847439a..6ac8c1d6603 100644 --- a/packages/server/src/utils/buildChatflow.ts +++ b/packages/server/src/utils/buildChatflow.ts @@ -9,7 +9,9 @@ import { mapMimeTypeToInputField, mapExtToInputField, generateFollowUpPrompts, - IServerSideEventStreamer + IServerSideEventStreamer, + removeSpecificFileFromUpload, + getFileFromUpload } from 'flowise-components' import { StatusCodes } from 'http-status-codes' import { @@ -49,7 +51,6 @@ import { validateChatflowAPIKey } from './validateKey' import { databaseEntities } from '.' import { v4 as uuidv4 } from 'uuid' import { omit } from 'lodash' -import * as fs from 'fs' import logger from './logger' import { utilAddChatMessage } from './addChatMesage' import { buildAgentGraph } from './buildAgentGraph' @@ -162,7 +163,7 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals const overrideConfig: ICommonObject = { ...req.body } const fileNames: string[] = [] for (const file of files) { - const fileBuffer = fs.readFileSync(file.path) + const fileBuffer = await getFileFromUpload(file.path ?? (file as any).key) // Address file name with special characters: https://github.com/expressjs/multer/issues/1104 file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8') const storagePath = await addArrayFilesToStorage(file.mimetype, fileBuffer, file.originalname, fileNames, chatflowid) @@ -195,7 +196,7 @@ export const utilBuildChatflow = async (req: Request, isInternal: boolean = fals overrideConfig[fileInputField] = storagePath } - fs.unlinkSync(file.path) + await removeSpecificFileFromUpload(file.path ?? (file as any).key) } if (overrideConfig.vars && typeof overrideConfig.vars === 'string') { overrideConfig.vars = JSON.parse(overrideConfig.vars) diff --git a/packages/server/src/utils/createAttachment.ts b/packages/server/src/utils/createAttachment.ts index 94b161ce3ca..4fa1bc95ed3 100644 --- a/packages/server/src/utils/createAttachment.ts +++ b/packages/server/src/utils/createAttachment.ts @@ -1,7 +1,13 @@ import { Request } from 'express' import * as path from 'path' -import * as fs from 'fs' -import { addArrayFilesToStorage, IDocument, mapExtToInputField, mapMimeTypeToInputField } from 'flowise-components' +import { + addArrayFilesToStorage, + getFileFromUpload, + removeSpecificFileFromUpload, + IDocument, + mapExtToInputField, + mapMimeTypeToInputField +} from 'flowise-components' import { getRunningExpressApp } from './getRunningExpressApp' import { getErrorMessage } from '../errors/utils' @@ -41,7 +47,7 @@ export const createFileAttachment = async (req: Request) => { if (files.length) { const isBase64 = req.body.base64 for (const file of files) { - const fileBuffer = fs.readFileSync(file.path) + const fileBuffer = await getFileFromUpload(file.path ?? (file as any).key) const fileNames: string[] = [] // Address file name with special characters: https://github.com/expressjs/multer/issues/1104 @@ -63,7 +69,7 @@ export const createFileAttachment = async (req: Request) => { fileInputField = fileInputFieldFromExt } - fs.unlinkSync(file.path) + await removeSpecificFileFromUpload(file.path ?? (file as any).key) try { const nodeData = { diff --git a/packages/server/src/utils/index.ts b/packages/server/src/utils/index.ts index 976ffb9e90d..d50012a6fd1 100644 --- a/packages/server/src/utils/index.ts +++ b/packages/server/src/utils/index.ts @@ -1690,21 +1690,6 @@ export const getTelemetryFlowObj = (nodes: IReactFlowNode[], edges: IReactFlowEd return { nodes: nodeData, edges: edgeData } } -/** - * Get user settings file - * TODO: move env variables to settings json file, easier configuration - */ -export const getUserSettingsFilePath = () => { - if (process.env.SECRETKEY_PATH) return path.join(process.env.SECRETKEY_PATH, 'settings.json') - const checkPaths = [path.join(getUserHome(), '.flowise', 'settings.json')] - for (const checkPath of checkPaths) { - if (fs.existsSync(checkPath)) { - return checkPath - } - } - return '' -} - /** * Get app current version */ @@ -1773,9 +1758,3 @@ export const getAPIOverrideConfig = (chatflow: IChatFlow) => { return { nodeOverrides: {}, variableOverrides: [], apiOverrideStatus: false } } } - -export const getUploadPath = (): string => { - return process.env.BLOB_STORAGE_PATH - ? path.join(process.env.BLOB_STORAGE_PATH, 'uploads') - : path.join(getUserHome(), '.flowise', 'uploads') -} diff --git a/packages/server/src/utils/telemetry.ts b/packages/server/src/utils/telemetry.ts index 4b033f2095b..07938f9f593 100644 --- a/packages/server/src/utils/telemetry.ts +++ b/packages/server/src/utils/telemetry.ts @@ -2,7 +2,8 @@ import { v4 as uuidv4 } from 'uuid' import { PostHog } from 'posthog-node' import path from 'path' import fs from 'fs' -import { getUserHome, getUserSettingsFilePath } from '.' +import { getUserHome } from '.' +import { getUserSettingsFilePath } from 'flowise-components' export class Telemetry { postHog?: PostHog diff --git a/packages/server/src/utils/upsertVector.ts b/packages/server/src/utils/upsertVector.ts index d92ab41d357..e490a21503c 100644 --- a/packages/server/src/utils/upsertVector.ts +++ b/packages/server/src/utils/upsertVector.ts @@ -1,8 +1,15 @@ import { Request } from 'express' -import * as fs from 'fs' import * as path from 'path' import { cloneDeep, omit } from 'lodash' -import { ICommonObject, IMessage, addArrayFilesToStorage, mapMimeTypeToInputField, mapExtToInputField } from 'flowise-components' +import { + ICommonObject, + IMessage, + addArrayFilesToStorage, + mapMimeTypeToInputField, + mapExtToInputField, + getFileFromUpload, + removeSpecificFileFromUpload +} from 'flowise-components' import logger from '../utils/logger' import { buildFlow, @@ -57,7 +64,7 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) => const overrideConfig: ICommonObject = { ...req.body } for (const file of files) { const fileNames: string[] = [] - const fileBuffer = fs.readFileSync(file.path) + const fileBuffer = await getFileFromUpload(file.path ?? (file as any).key) // Address file name with special characters: https://github.com/expressjs/multer/issues/1104 file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8') const storagePath = await addArrayFilesToStorage(file.mimetype, fileBuffer, file.originalname, fileNames, chatflowid) @@ -90,7 +97,7 @@ export const upsertVector = async (req: Request, isInternal: boolean = false) => overrideConfig[fileInputField] = storagePath } - fs.unlinkSync(file.path) + await removeSpecificFileFromUpload(file.path ?? (file as any).key) } if (overrideConfig.vars && typeof overrideConfig.vars === 'string') { overrideConfig.vars = JSON.parse(overrideConfig.vars) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 20c0bb6e105..26d8b06feba 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -168,7 +168,7 @@ importers: version: 0.0.7(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)) '@langchain/community': specifier: ^0.3.11 - version: 0.3.14(@aws-crypto/sha256-js@5.2.0)(@aws-sdk/client-bedrock-agent-runtime@3.625.0)(@aws-sdk/client-bedrock-runtime@3.422.0)(@aws-sdk/client-dynamodb@3.529.1)(@aws-sdk/client-kendra@3.624.0)(@aws-sdk/client-s3@3.529.1)(@aws-sdk/credential-provider-node@3.529.1)(@datastax/astra-db-ts@1.5.0)(@elastic/elasticsearch@8.12.2)(@getzep/zep-cloud@1.0.7(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13)(langchain@0.3.5(@langchain/anthropic@0.3.7(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/aws@0.1.2(@aws-sdk/client-sso-oidc@3.723.0(@aws-sdk/client-sts@3.723.0))(@aws-sdk/client-sts@3.624.0)(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(@langchain/cohere@0.0.7(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/google-genai@0.1.3(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(zod@3.22.4))(@langchain/google-vertexai@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13)(zod@3.22.4))(@langchain/groq@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/mistralai@0.0.26(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/ollama@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(axios@1.6.2)(cheerio@1.0.0-rc.12)(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))(typeorm@0.3.20(ioredis@5.3.2)(mongodb@6.3.0(gcp-metadata@6.1.0(encoding@0.1.13))(socks@2.8.1))(mysql2@3.11.4)(pg@8.11.3)(redis@4.6.13)(sqlite3@5.1.7)(ts-node@10.9.2(@swc/core@1.4.6)(@types/node@20.12.12)(typescript@5.5.2)))))(@getzep/zep-js@0.9.0)(@gomomento/sdk-core@1.68.1)(@gomomento/sdk@1.68.1(encoding@0.1.13))(@google-ai/generativelanguage@2.6.0(encoding@0.1.13))(@huggingface/inference@2.6.4)(@ibm-cloud/watsonx-ai@1.1.2)(@langchain/anthropic@0.3.7(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/aws@0.1.2(@aws-sdk/client-sso-oidc@3.723.0(@aws-sdk/client-sts@3.723.0))(@aws-sdk/client-sts@3.624.0)(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(@langchain/cohere@0.0.7(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/google-genai@0.1.3(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(zod@3.22.4))(@langchain/google-vertexai@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13)(zod@3.22.4))(@langchain/groq@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/mistralai@0.0.26(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/ollama@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(@mendable/firecrawl-js@0.0.28)(@notionhq/client@2.2.14(encoding@0.1.13))(@opensearch-project/opensearch@1.2.0)(@pinecone-database/pinecone@4.0.0)(@qdrant/js-client-rest@1.9.0(typescript@5.5.2))(@smithy/eventstream-codec@3.1.2)(@smithy/protocol-http@4.1.0)(@smithy/signature-v4@4.1.0)(@smithy/util-utf8@3.0.0)(@supabase/supabase-js@2.39.8(bufferutil@4.0.8)(utf-8-validate@6.0.4))(@upstash/redis@1.22.1(encoding@0.1.13))(@upstash/vector@1.1.5)(@xenova/transformers@2.17.1)(@zilliz/milvus2-sdk-node@2.3.5)(apify-client@2.9.3)(assemblyai@4.3.2(bufferutil@4.0.8)(utf-8-validate@6.0.4))(axios@1.6.2)(cheerio@1.0.0-rc.12)(chromadb@1.8.1(@google/generative-ai@0.15.0)(cohere-ai@7.10.0(encoding@0.1.13))(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(cohere-ai@7.10.0(encoding@0.1.13))(crypto-js@4.2.0)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(faiss-node@0.5.1)(google-auth-library@9.6.3(encoding@0.1.13))(html-to-text@9.0.5)(ibm-cloud-sdk-core@5.1.0)(ignore@5.3.1)(ioredis@5.3.2)(jsdom@22.1.0(bufferutil@4.0.8)(canvas@2.11.2(encoding@0.1.13))(utf-8-validate@6.0.4))(jsonwebtoken@9.0.2)(lodash@4.17.21)(lunary@0.7.12(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))(react@18.2.0))(mammoth@1.7.0)(mongodb@6.3.0(gcp-metadata@6.1.0(encoding@0.1.13))(socks@2.8.1))(mysql2@3.11.4)(neo4j-driver@5.27.0)(notion-to-md@3.1.1(encoding@0.1.13))(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))(pdf-parse@1.1.1)(pg@8.11.3)(playwright@1.42.1)(portkey-ai@0.1.16)(puppeteer@20.9.0(bufferutil@4.0.8)(encoding@0.1.13)(typescript@5.5.2)(utf-8-validate@6.0.4))(pyodide@0.25.0(bufferutil@4.0.8)(utf-8-validate@6.0.4))(redis@4.6.13)(replicate@0.31.1)(srt-parser-2@1.2.3)(typeorm@0.3.20(ioredis@5.3.2)(mongodb@6.3.0(gcp-metadata@6.1.0(encoding@0.1.13))(socks@2.8.1))(mysql2@3.11.4)(pg@8.11.3)(redis@4.6.13)(sqlite3@5.1.7)(ts-node@10.9.2(@swc/core@1.4.6)(@types/node@20.12.12)(typescript@5.5.2)))(weaviate-ts-client@1.6.0(encoding@0.1.13)(graphql@16.8.1))(ws@8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.4)) + version: 0.3.14(@aws-crypto/sha256-js@5.2.0)(@aws-sdk/client-bedrock-agent-runtime@3.625.0)(@aws-sdk/client-bedrock-runtime@3.422.0)(@aws-sdk/client-dynamodb@3.529.1)(@aws-sdk/client-kendra@3.624.0)(@aws-sdk/client-s3@3.529.1)(@aws-sdk/credential-provider-node@3.529.1)(@datastax/astra-db-ts@1.5.0)(@elastic/elasticsearch@8.12.2)(@getzep/zep-cloud@1.0.7(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13)(langchain@0.3.5(@langchain/anthropic@0.3.7(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/aws@0.1.2(@aws-sdk/client-sso-oidc@3.723.0(@aws-sdk/client-sts@3.723.0))(@aws-sdk/client-sts@3.624.0)(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(@langchain/cohere@0.0.7(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/google-genai@0.1.3(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(zod@3.22.4))(@langchain/google-vertexai@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13)(zod@3.22.4))(@langchain/groq@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/mistralai@0.0.26(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/ollama@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(axios@1.6.2)(cheerio@1.0.0-rc.12)(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))(typeorm@0.3.20(ioredis@5.3.2)(mongodb@6.3.0(gcp-metadata@6.1.0(encoding@0.1.13))(socks@2.8.1))(mysql2@3.11.4)(pg@8.11.3)(redis@4.6.13)(sqlite3@5.1.7)(ts-node@10.9.2(@swc/core@1.4.6)(@types/node@20.12.12)(typescript@5.5.2)))))(@getzep/zep-js@0.9.0)(@gomomento/sdk-core@1.68.1)(@gomomento/sdk@1.68.1(encoding@0.1.13))(@google-ai/generativelanguage@2.6.0(encoding@0.1.13))(@huggingface/inference@2.6.4)(@ibm-cloud/watsonx-ai@1.1.2)(@langchain/anthropic@0.3.7(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/aws@0.1.2(@aws-sdk/client-sso-oidc@3.723.0(@aws-sdk/client-sts@3.723.0))(@aws-sdk/client-sts@3.624.0)(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(@langchain/cohere@0.0.7(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/google-genai@0.1.3(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(zod@3.22.4))(@langchain/google-vertexai@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13)(zod@3.22.4))(@langchain/groq@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/mistralai@0.0.26(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/ollama@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(@mendable/firecrawl-js@0.0.28)(@notionhq/client@2.2.14(encoding@0.1.13))(@opensearch-project/opensearch@1.2.0)(@pinecone-database/pinecone@4.0.0)(@qdrant/js-client-rest@1.9.0(typescript@5.5.2))(@smithy/eventstream-codec@3.1.2)(@smithy/protocol-http@5.0.0)(@smithy/signature-v4@5.0.0)(@smithy/util-utf8@4.0.0)(@supabase/supabase-js@2.39.8(bufferutil@4.0.8)(utf-8-validate@6.0.4))(@upstash/redis@1.22.1(encoding@0.1.13))(@upstash/vector@1.1.5)(@xenova/transformers@2.17.1)(@zilliz/milvus2-sdk-node@2.3.5)(apify-client@2.9.3)(assemblyai@4.3.2(bufferutil@4.0.8)(utf-8-validate@6.0.4))(axios@1.6.2)(cheerio@1.0.0-rc.12)(chromadb@1.8.1(@google/generative-ai@0.15.0)(cohere-ai@7.10.0(encoding@0.1.13))(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(cohere-ai@7.10.0(encoding@0.1.13))(crypto-js@4.2.0)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(faiss-node@0.5.1)(google-auth-library@9.6.3(encoding@0.1.13))(html-to-text@9.0.5)(ibm-cloud-sdk-core@5.1.0)(ignore@5.3.1)(ioredis@5.3.2)(jsdom@22.1.0(bufferutil@4.0.8)(canvas@2.11.2(encoding@0.1.13))(utf-8-validate@6.0.4))(jsonwebtoken@9.0.2)(lodash@4.17.21)(lunary@0.7.12(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))(react@18.2.0))(mammoth@1.7.0)(mongodb@6.3.0(gcp-metadata@6.1.0(encoding@0.1.13))(socks@2.8.1))(mysql2@3.11.4)(neo4j-driver@5.27.0)(notion-to-md@3.1.1(encoding@0.1.13))(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))(pdf-parse@1.1.1)(pg@8.11.3)(playwright@1.42.1)(portkey-ai@0.1.16)(puppeteer@20.9.0(bufferutil@4.0.8)(encoding@0.1.13)(typescript@5.5.2)(utf-8-validate@6.0.4))(pyodide@0.25.0(bufferutil@4.0.8)(utf-8-validate@6.0.4))(redis@4.6.13)(replicate@0.31.1)(srt-parser-2@1.2.3)(typeorm@0.3.20(ioredis@5.3.2)(mongodb@6.3.0(gcp-metadata@6.1.0(encoding@0.1.13))(socks@2.8.1))(mysql2@3.11.4)(pg@8.11.3)(redis@4.6.13)(sqlite3@5.1.7)(ts-node@10.9.2(@swc/core@1.4.6)(@types/node@20.12.12)(typescript@5.5.2)))(weaviate-ts-client@1.6.0(encoding@0.1.13)(graphql@16.8.1))(ws@8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.4)) '@langchain/core': specifier: 0.3.18 version: 0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)) @@ -596,6 +596,9 @@ importers: multer: specifier: ^1.4.5-lts.1 version: 1.4.5-lts.1 + multer-s3: + specifier: ^3.0.1 + version: 3.0.1(@aws-sdk/client-s3@3.529.1) mysql2: specifier: ^3.11.3 version: 3.11.4 @@ -645,6 +648,9 @@ importers: '@types/multer': specifier: ^1.4.7 version: 1.4.11 + '@types/multer-s3': + specifier: ^3.0.3 + version: 3.0.3 '@types/sanitize-html': specifier: ^2.9.5 version: 2.11.0 @@ -1261,6 +1267,12 @@ packages: resolution: { integrity: sha512-XCDrpiS50WaPzPzp7FwsChPHtX9PQQUU4nRzcn2N7IkUtpcFCUx8m1PAZe086VQr6hrbdeE4Z4j8hUPNwVdJGQ== } engines: { node: '>=14.0.0' } + '@aws-sdk/lib-storage@3.726.1': + resolution: { integrity: sha512-WuDxSZ8Bfe1N7gn5eXQ02dhlKWCAwW5qQErpJ4CCddXosF+gLxhGkrP9LkaaP0CpA3PxboHyET6HbWAggOWtqA== } + engines: { node: '>=18.0.0' } + peerDependencies: + '@aws-sdk/client-s3': ^3.726.1 + '@aws-sdk/middleware-bucket-endpoint@3.525.0': resolution: { integrity: sha512-nYfQ2Xspfef7j8mZO7varUWLPH6HQlXateH7tBVtBNUAazyQE4UJEvC0fbQ+Y01e+FKlirim/m2umkdMXqAlTg== } engines: { node: '>=14.0.0' } @@ -6174,6 +6186,9 @@ packages: '@types/ms@0.7.34': resolution: { integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g== } + '@types/multer-s3@3.0.3': + resolution: { integrity: sha512-VgWygI9UwyS7loLithUUi0qAMIDWdNrERS2Sb06UuPYiLzKuIFn2NgL7satyl4v8sh/LLoU7DiPanvbQaRg9Yg== } + '@types/multer@1.4.11': resolution: { integrity: sha512-svK240gr6LVWvv3YGyhLlA+6LRRWA4mnGIU7RcNmgjBYFl6665wcXrRfxGp5tEPVHUNm5FMcmq7too9bxCwX/w== } @@ -7401,6 +7416,9 @@ packages: buffer@4.9.2: resolution: { integrity: sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg== } + buffer@5.6.0: + resolution: { integrity: sha512-/gDYp/UtU0eA1ys8bOs9J6a+E/KWIY+DZ+Q2WESNUA0jFRsJOc0SNUO6xJ5SGA1xueg3NL65W6s+NY5l9cunuw== } + buffer@5.7.1: resolution: { integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ== } @@ -9303,6 +9321,10 @@ packages: resolution: { integrity: sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw== } engines: { node: '>=10' } + file-type@3.9.0: + resolution: { integrity: sha512-RLoqTXE8/vPmMuTI88DAzhMYC99I8BWv7zYP4A1puo5HIjEJ5EX48ighy4ZyKMG9EDXxBgW6e++cn7d1xuFghA== } + engines: { node: '>=0.10.0' } + file-uri-to-path@1.0.0: resolution: { integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw== } @@ -10053,6 +10075,9 @@ packages: resolution: { integrity: sha512-A91dYTeIB6NoXG+PxTQpCCDDnfHsW9kc06Lvpu1TEe9gnd6ZFeiBoRO9JvzEv6xK7EX97/dUE8g/vBMTqTS3CA== } engines: { node: '>=14' } + html-comment-regex@1.1.2: + resolution: { integrity: sha512-P+M65QY2JQ5Y0G9KKdlDpo0zK+/OHptU5AaBwUfAIDJZk1MYf32Frm84EcOytfJE0t5JvkAnKlmjsXDnWzCJmQ== } + html-dom-parser@3.1.7: resolution: { integrity: sha512-cDgNF4YgF6J3H+d9mcldGL19p0GzVdS3iGuDNzYWQpU47q3+IRM85X3Xo07E+nntF4ek4s78A9V24EwxlPTjig== } @@ -12048,6 +12073,12 @@ packages: ms@2.1.3: resolution: { integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA== } + multer-s3@3.0.1: + resolution: { integrity: sha512-BFwSO80a5EW4GJRBdUuSHblz2jhVSAze33ZbnGpcfEicoT0iRolx4kWR+AJV07THFRCQ78g+kelKFdjkCCaXeQ== } + engines: { node: '>= 12.0.0' } + peerDependencies: + '@aws-sdk/client-s3': ^3.0.0 + multer@1.4.5-lts.1: resolution: { integrity: sha512-ywPWvcDMeH+z9gQq5qYHCCy+ethsk4goepZ45GLD63fOu0YcNecQxi64nDs3qluZB+murG3/D4dJ7+dGctcCQQ== } engines: { node: '>= 6.0.0' } @@ -14867,6 +14898,9 @@ packages: resolution: { integrity: sha512-iCGQj+0l0HOdZ2AEeBADlsRC+vsnDsZsbdSiH1yNSjcfKM7fdpCMfqAL/dwF5BLiw/XhRft/Wax6zQbhq2BcjQ== } engines: { node: '>= 0.4' } + stream-browserify@3.0.0: + resolution: { integrity: sha512-H73RAHsVBapbim0tU2JwwOiXUj+fikfiaoYAKHF3VJfA0pe2BCzkhAHBlLG6REzE+2WNZcxOXjK7lkso+9euLA== } + stream-combiner@0.0.4: resolution: { integrity: sha512-rT00SPnTVyRsaSz5zgSPma/aHSOic5U1prhYdRy5HS2kTZviFpmDgzilbtsJsxiroqACmayynDN/9VzIbX5DOw== } @@ -16848,19 +16882,19 @@ snapshots: '@aws-crypto/crc32@3.0.0': dependencies: '@aws-crypto/util': 3.0.0 - '@aws-sdk/types': 3.609.0 + '@aws-sdk/types': 3.723.0 tslib: 1.14.1 '@aws-crypto/crc32@5.2.0': dependencies: '@aws-crypto/util': 5.2.0 - '@aws-sdk/types': 3.609.0 + '@aws-sdk/types': 3.723.0 tslib: 2.6.2 '@aws-crypto/crc32c@3.0.0': dependencies: '@aws-crypto/util': 3.0.0 - '@aws-sdk/types': 3.609.0 + '@aws-sdk/types': 3.723.0 tslib: 1.14.1 '@aws-crypto/ie11-detection@3.0.0': @@ -16920,7 +16954,7 @@ snapshots: '@aws-crypto/util@3.0.0': dependencies: - '@aws-sdk/types': 3.609.0 + '@aws-sdk/types': 3.723.0 '@aws-sdk/util-utf8-browser': 3.259.0 tslib: 1.14.1 @@ -18222,6 +18256,17 @@ snapshots: mnemonist: 0.38.3 tslib: 2.6.2 + '@aws-sdk/lib-storage@3.726.1(@aws-sdk/client-s3@3.529.1)': + dependencies: + '@aws-sdk/client-s3': 3.529.1 + '@smithy/abort-controller': 4.0.0 + '@smithy/middleware-endpoint': 4.0.0 + '@smithy/smithy-client': 4.0.0 + buffer: 5.6.0 + events: 3.3.0 + stream-browserify: 3.0.0 + tslib: 2.6.2 + '@aws-sdk/middleware-bucket-endpoint@3.525.0': dependencies: '@aws-sdk/types': 3.523.0 @@ -21104,7 +21149,7 @@ snapshots: - encoding - openai - '@langchain/community@0.3.14(@aws-crypto/sha256-js@5.2.0)(@aws-sdk/client-bedrock-agent-runtime@3.625.0)(@aws-sdk/client-bedrock-runtime@3.422.0)(@aws-sdk/client-dynamodb@3.529.1)(@aws-sdk/client-kendra@3.624.0)(@aws-sdk/client-s3@3.529.1)(@aws-sdk/credential-provider-node@3.529.1)(@datastax/astra-db-ts@1.5.0)(@elastic/elasticsearch@8.12.2)(@getzep/zep-cloud@1.0.7(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13)(langchain@0.3.5(@langchain/anthropic@0.3.7(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/aws@0.1.2(@aws-sdk/client-sso-oidc@3.723.0(@aws-sdk/client-sts@3.723.0))(@aws-sdk/client-sts@3.624.0)(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(@langchain/cohere@0.0.7(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/google-genai@0.1.3(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(zod@3.22.4))(@langchain/google-vertexai@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13)(zod@3.22.4))(@langchain/groq@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/mistralai@0.0.26(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/ollama@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(axios@1.6.2)(cheerio@1.0.0-rc.12)(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))(typeorm@0.3.20(ioredis@5.3.2)(mongodb@6.3.0(gcp-metadata@6.1.0(encoding@0.1.13))(socks@2.8.1))(mysql2@3.11.4)(pg@8.11.3)(redis@4.6.13)(sqlite3@5.1.7)(ts-node@10.9.2(@swc/core@1.4.6)(@types/node@20.12.12)(typescript@5.5.2)))))(@getzep/zep-js@0.9.0)(@gomomento/sdk-core@1.68.1)(@gomomento/sdk@1.68.1(encoding@0.1.13))(@google-ai/generativelanguage@2.6.0(encoding@0.1.13))(@huggingface/inference@2.6.4)(@ibm-cloud/watsonx-ai@1.1.2)(@langchain/anthropic@0.3.7(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/aws@0.1.2(@aws-sdk/client-sso-oidc@3.723.0(@aws-sdk/client-sts@3.723.0))(@aws-sdk/client-sts@3.624.0)(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(@langchain/cohere@0.0.7(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/google-genai@0.1.3(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(zod@3.22.4))(@langchain/google-vertexai@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13)(zod@3.22.4))(@langchain/groq@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/mistralai@0.0.26(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/ollama@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(@mendable/firecrawl-js@0.0.28)(@notionhq/client@2.2.14(encoding@0.1.13))(@opensearch-project/opensearch@1.2.0)(@pinecone-database/pinecone@4.0.0)(@qdrant/js-client-rest@1.9.0(typescript@5.5.2))(@smithy/eventstream-codec@3.1.2)(@smithy/protocol-http@4.1.0)(@smithy/signature-v4@4.1.0)(@smithy/util-utf8@3.0.0)(@supabase/supabase-js@2.39.8(bufferutil@4.0.8)(utf-8-validate@6.0.4))(@upstash/redis@1.22.1(encoding@0.1.13))(@upstash/vector@1.1.5)(@xenova/transformers@2.17.1)(@zilliz/milvus2-sdk-node@2.3.5)(apify-client@2.9.3)(assemblyai@4.3.2(bufferutil@4.0.8)(utf-8-validate@6.0.4))(axios@1.6.2)(cheerio@1.0.0-rc.12)(chromadb@1.8.1(@google/generative-ai@0.15.0)(cohere-ai@7.10.0(encoding@0.1.13))(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(cohere-ai@7.10.0(encoding@0.1.13))(crypto-js@4.2.0)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(faiss-node@0.5.1)(google-auth-library@9.6.3(encoding@0.1.13))(html-to-text@9.0.5)(ibm-cloud-sdk-core@5.1.0)(ignore@5.3.1)(ioredis@5.3.2)(jsdom@22.1.0(bufferutil@4.0.8)(canvas@2.11.2(encoding@0.1.13))(utf-8-validate@6.0.4))(jsonwebtoken@9.0.2)(lodash@4.17.21)(lunary@0.7.12(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))(react@18.2.0))(mammoth@1.7.0)(mongodb@6.3.0(gcp-metadata@6.1.0(encoding@0.1.13))(socks@2.8.1))(mysql2@3.11.4)(neo4j-driver@5.27.0)(notion-to-md@3.1.1(encoding@0.1.13))(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))(pdf-parse@1.1.1)(pg@8.11.3)(playwright@1.42.1)(portkey-ai@0.1.16)(puppeteer@20.9.0(bufferutil@4.0.8)(encoding@0.1.13)(typescript@5.5.2)(utf-8-validate@6.0.4))(pyodide@0.25.0(bufferutil@4.0.8)(utf-8-validate@6.0.4))(redis@4.6.13)(replicate@0.31.1)(srt-parser-2@1.2.3)(typeorm@0.3.20(ioredis@5.3.2)(mongodb@6.3.0(gcp-metadata@6.1.0(encoding@0.1.13))(socks@2.8.1))(mysql2@3.11.4)(pg@8.11.3)(redis@4.6.13)(sqlite3@5.1.7)(ts-node@10.9.2(@swc/core@1.4.6)(@types/node@20.12.12)(typescript@5.5.2)))(weaviate-ts-client@1.6.0(encoding@0.1.13)(graphql@16.8.1))(ws@8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.4))': + '@langchain/community@0.3.14(@aws-crypto/sha256-js@5.2.0)(@aws-sdk/client-bedrock-agent-runtime@3.625.0)(@aws-sdk/client-bedrock-runtime@3.422.0)(@aws-sdk/client-dynamodb@3.529.1)(@aws-sdk/client-kendra@3.624.0)(@aws-sdk/client-s3@3.529.1)(@aws-sdk/credential-provider-node@3.529.1)(@datastax/astra-db-ts@1.5.0)(@elastic/elasticsearch@8.12.2)(@getzep/zep-cloud@1.0.7(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13)(langchain@0.3.5(@langchain/anthropic@0.3.7(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/aws@0.1.2(@aws-sdk/client-sso-oidc@3.723.0(@aws-sdk/client-sts@3.723.0))(@aws-sdk/client-sts@3.624.0)(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(@langchain/cohere@0.0.7(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/google-genai@0.1.3(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(zod@3.22.4))(@langchain/google-vertexai@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13)(zod@3.22.4))(@langchain/groq@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/mistralai@0.0.26(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/ollama@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(axios@1.6.2)(cheerio@1.0.0-rc.12)(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))(typeorm@0.3.20(ioredis@5.3.2)(mongodb@6.3.0(gcp-metadata@6.1.0(encoding@0.1.13))(socks@2.8.1))(mysql2@3.11.4)(pg@8.11.3)(redis@4.6.13)(sqlite3@5.1.7)(ts-node@10.9.2(@swc/core@1.4.6)(@types/node@20.12.12)(typescript@5.5.2)))))(@getzep/zep-js@0.9.0)(@gomomento/sdk-core@1.68.1)(@gomomento/sdk@1.68.1(encoding@0.1.13))(@google-ai/generativelanguage@2.6.0(encoding@0.1.13))(@huggingface/inference@2.6.4)(@ibm-cloud/watsonx-ai@1.1.2)(@langchain/anthropic@0.3.7(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/aws@0.1.2(@aws-sdk/client-sso-oidc@3.723.0(@aws-sdk/client-sts@3.723.0))(@aws-sdk/client-sts@3.624.0)(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(@langchain/cohere@0.0.7(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/google-genai@0.1.3(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(zod@3.22.4))(@langchain/google-vertexai@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13)(zod@3.22.4))(@langchain/groq@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(encoding@0.1.13))(@langchain/mistralai@0.0.26(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(@langchain/ollama@0.1.2(@langchain/core@0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))))(@mendable/firecrawl-js@0.0.28)(@notionhq/client@2.2.14(encoding@0.1.13))(@opensearch-project/opensearch@1.2.0)(@pinecone-database/pinecone@4.0.0)(@qdrant/js-client-rest@1.9.0(typescript@5.5.2))(@smithy/eventstream-codec@3.1.2)(@smithy/protocol-http@5.0.0)(@smithy/signature-v4@5.0.0)(@smithy/util-utf8@4.0.0)(@supabase/supabase-js@2.39.8(bufferutil@4.0.8)(utf-8-validate@6.0.4))(@upstash/redis@1.22.1(encoding@0.1.13))(@upstash/vector@1.1.5)(@xenova/transformers@2.17.1)(@zilliz/milvus2-sdk-node@2.3.5)(apify-client@2.9.3)(assemblyai@4.3.2(bufferutil@4.0.8)(utf-8-validate@6.0.4))(axios@1.6.2)(cheerio@1.0.0-rc.12)(chromadb@1.8.1(@google/generative-ai@0.15.0)(cohere-ai@7.10.0(encoding@0.1.13))(encoding@0.1.13)(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)))(cohere-ai@7.10.0(encoding@0.1.13))(crypto-js@4.2.0)(d3-dsv@2.0.0)(encoding@0.1.13)(epub2@3.0.2(ts-toolbelt@9.6.0))(faiss-node@0.5.1)(google-auth-library@9.6.3(encoding@0.1.13))(html-to-text@9.0.5)(ibm-cloud-sdk-core@5.1.0)(ignore@5.3.1)(ioredis@5.3.2)(jsdom@22.1.0(bufferutil@4.0.8)(canvas@2.11.2(encoding@0.1.13))(utf-8-validate@6.0.4))(jsonwebtoken@9.0.2)(lodash@4.17.21)(lunary@0.7.12(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))(react@18.2.0))(mammoth@1.7.0)(mongodb@6.3.0(gcp-metadata@6.1.0(encoding@0.1.13))(socks@2.8.1))(mysql2@3.11.4)(neo4j-driver@5.27.0)(notion-to-md@3.1.1(encoding@0.1.13))(openai@4.57.3(encoding@0.1.13)(zod@3.22.4))(pdf-parse@1.1.1)(pg@8.11.3)(playwright@1.42.1)(portkey-ai@0.1.16)(puppeteer@20.9.0(bufferutil@4.0.8)(encoding@0.1.13)(typescript@5.5.2)(utf-8-validate@6.0.4))(pyodide@0.25.0(bufferutil@4.0.8)(utf-8-validate@6.0.4))(redis@4.6.13)(replicate@0.31.1)(srt-parser-2@1.2.3)(typeorm@0.3.20(ioredis@5.3.2)(mongodb@6.3.0(gcp-metadata@6.1.0(encoding@0.1.13))(socks@2.8.1))(mysql2@3.11.4)(pg@8.11.3)(redis@4.6.13)(sqlite3@5.1.7)(ts-node@10.9.2(@swc/core@1.4.6)(@types/node@20.12.12)(typescript@5.5.2)))(weaviate-ts-client@1.6.0(encoding@0.1.13)(graphql@16.8.1))(ws@8.16.0(bufferutil@4.0.8)(utf-8-validate@6.0.4))': dependencies: '@ibm-cloud/watsonx-ai': 1.1.2 '@langchain/core': 0.3.18(openai@4.57.3(encoding@0.1.13)(zod@3.22.4)) @@ -21141,9 +21186,9 @@ snapshots: '@pinecone-database/pinecone': 4.0.0 '@qdrant/js-client-rest': 1.9.0(typescript@5.5.2) '@smithy/eventstream-codec': 3.1.2 - '@smithy/protocol-http': 4.1.0 - '@smithy/signature-v4': 4.1.0 - '@smithy/util-utf8': 3.0.0 + '@smithy/protocol-http': 5.0.0 + '@smithy/signature-v4': 5.0.0 + '@smithy/util-utf8': 4.0.0 '@supabase/supabase-js': 2.39.8(bufferutil@4.0.8)(utf-8-validate@6.0.4) '@upstash/redis': 1.22.1(encoding@0.1.13) '@upstash/vector': 1.1.5 @@ -24410,6 +24455,14 @@ snapshots: '@types/ms@0.7.34': {} + '@types/multer-s3@3.0.3': + dependencies: + '@aws-sdk/client-s3': 3.529.1 + '@types/multer': 1.4.11 + '@types/node': 20.12.12 + transitivePeerDependencies: + - aws-crt + '@types/multer@1.4.11': dependencies: '@types/express': 4.17.21 @@ -26039,6 +26092,11 @@ snapshots: ieee754: 1.2.1 isarray: 1.0.0 + buffer@5.6.0: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + buffer@5.7.1: dependencies: base64-js: 1.5.1 @@ -28443,6 +28501,8 @@ snapshots: strtok3: 6.3.0 token-types: 4.2.1 + file-type@3.9.0: {} + file-uri-to-path@1.0.0: {} filelist@1.0.4: @@ -29493,6 +29553,8 @@ snapshots: hpagent@1.2.0: {} + html-comment-regex@1.1.2: {} + html-dom-parser@3.1.7: dependencies: domhandler: 5.0.3 @@ -32148,6 +32210,14 @@ snapshots: ms@2.1.3: {} + multer-s3@3.0.1(@aws-sdk/client-s3@3.529.1): + dependencies: + '@aws-sdk/client-s3': 3.529.1 + '@aws-sdk/lib-storage': 3.726.1(@aws-sdk/client-s3@3.529.1) + file-type: 3.9.0 + html-comment-regex: 1.1.2 + run-parallel: 1.2.0 + multer@1.4.5-lts.1: dependencies: append-field: 1.0.0 @@ -35530,6 +35600,11 @@ snapshots: dependencies: internal-slot: 1.0.7 + stream-browserify@3.0.0: + dependencies: + inherits: 2.0.4 + readable-stream: 3.6.2 + stream-combiner@0.0.4: dependencies: duplexer: 0.1.2 From 2dabdf8aeabee53561fbb6037acc6cd6588706c9 Mon Sep 17 00:00:00 2001 From: Henry Date: Sun, 12 Jan 2025 00:04:14 +0000 Subject: [PATCH 2/2] update multers3 --- packages/components/src/storageUtils.ts | 6 +++--- packages/server/package.json | 1 - pnpm-lock.yaml | 14 -------------- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/packages/components/src/storageUtils.ts b/packages/components/src/storageUtils.ts index 6b365f0d9eb..8f72f9a590b 100644 --- a/packages/components/src/storageUtils.ts +++ b/packages/components/src/storageUtils.ts @@ -12,7 +12,7 @@ import { Readable } from 'node:stream' import { getUserHome } from './utils' import sanitize from 'sanitize-filename' import multer from 'multer' -import multerS3 from 'multer-s3' +const multerS3 = require('multer-s3') /** * Get user settings file @@ -56,10 +56,10 @@ export const getMulterStorage = () => { storage: multerS3({ s3: s3Client, bucket: Bucket, - metadata: function (req, file, cb) { + metadata: function (req: Request, file: Express.Multer.File, cb: (error: any, metadata: any) => void) { cb(null, { fieldName: file.fieldname, originalName: file.originalname, orgId: getOrgId() }) }, - key: function (req, file, cb) { + key: function (req: Request, file: Express.Multer.File, cb: (error: any, metadata: any) => void) { cb(null, `${getOrgId()}/${Date.now().toString()}`) } }) diff --git a/packages/server/package.json b/packages/server/package.json index 8dceeeb4429..71ed3ad6f18 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -111,7 +111,6 @@ "@types/cors": "^2.8.12", "@types/crypto-js": "^4.1.1", "@types/multer": "^1.4.7", - "@types/multer-s3": "^3.0.3", "@types/sanitize-html": "^2.9.5", "concurrently": "^7.1.0", "cypress": "^13.13.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4e0cba394a1..61bab695b2c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -651,9 +651,6 @@ importers: '@types/multer': specifier: ^1.4.7 version: 1.4.11 - '@types/multer-s3': - specifier: ^3.0.3 - version: 3.0.3 '@types/sanitize-html': specifier: ^2.9.5 version: 2.11.0 @@ -6189,9 +6186,6 @@ packages: '@types/ms@0.7.34': resolution: { integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g== } - '@types/multer-s3@3.0.3': - resolution: { integrity: sha512-VgWygI9UwyS7loLithUUi0qAMIDWdNrERS2Sb06UuPYiLzKuIFn2NgL7satyl4v8sh/LLoU7DiPanvbQaRg9Yg== } - '@types/multer@1.4.11': resolution: { integrity: sha512-svK240gr6LVWvv3YGyhLlA+6LRRWA4mnGIU7RcNmgjBYFl6665wcXrRfxGp5tEPVHUNm5FMcmq7too9bxCwX/w== } @@ -24465,14 +24459,6 @@ snapshots: '@types/ms@0.7.34': {} - '@types/multer-s3@3.0.3': - dependencies: - '@aws-sdk/client-s3': 3.529.1 - '@types/multer': 1.4.11 - '@types/node': 20.12.12 - transitivePeerDependencies: - - aws-crt - '@types/multer@1.4.11': dependencies: '@types/express': 4.17.21