-
Notifications
You must be signed in to change notification settings - Fork 40
feat(filesystem): S3-backed filesystem support for agents #2745
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
viktormarinho
wants to merge
5
commits into
main
Choose a base branch
from
viktormarinho/object-storage
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
5 commits
Select commit
Hold shift + click to select a range
87b3045
feat(filesystem): add first-class S3-backed filesystem support
viktormarinho d46c186
merge: resolve conflict in env.ts after main merge
viktormarinho 5af6464
fix(filesystem): remove unused barrel export flagged by knip
viktormarinho 1f08174
merge: resolve conflict in env.ts after main merge
viktormarinho cb0b273
merge: resolve conflicts in package.json and bun.lock after main merge
viktormarinho File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -12,7 +12,7 @@ import { env } from "../env"; | |||||||||||||||||||
| import { DECO_STORE_URL, isDecoHostedMcp } from "@/core/deco-constants"; | ||||||||||||||||||||
| import { WellKnownOrgMCPId } from "@decocms/mesh-sdk"; | ||||||||||||||||||||
| import { PrometheusSerializer } from "@opentelemetry/exporter-prometheus"; | ||||||||||||||||||||
| import { Hono } from "hono"; | ||||||||||||||||||||
| import { type Context, Hono } from "hono"; | ||||||||||||||||||||
| import { getCookie } from "hono/cookie"; | ||||||||||||||||||||
| import { cors } from "hono/cors"; | ||||||||||||||||||||
| import { logger } from "hono/logger"; | ||||||||||||||||||||
|
|
@@ -956,6 +956,47 @@ export async function createApp(options: CreateAppOptions = {}) { | |||||||||||||||||||
| mountDevRoutes(app, mcpAuth); | ||||||||||||||||||||
| } | ||||||||||||||||||||
|
|
||||||||||||||||||||
| // Filesystem MCP routes (S3-backed filesystem for agents) | ||||||||||||||||||||
| // Handle /mcp/filesystem alias | ||||||||||||||||||||
| app.use("/mcp/filesystem", mcpAuth); | ||||||||||||||||||||
| app.route( | ||||||||||||||||||||
| "/mcp/filesystem", | ||||||||||||||||||||
| (await import("./routes/filesystem-mcp")).default, | ||||||||||||||||||||
| ); | ||||||||||||||||||||
|
|
||||||||||||||||||||
| // Handle {org_id}_filesystem connection ID pattern | ||||||||||||||||||||
| app.all( | ||||||||||||||||||||
| "/mcp/:connectionId{.*_filesystem$}", | ||||||||||||||||||||
| mcpAuth, | ||||||||||||||||||||
| async (c: Context<Env>) => { | ||||||||||||||||||||
| const ctx = c.get("meshContext") as MeshContext; | ||||||||||||||||||||
| const { handleFilesystemMcpRequest } = await import( | ||||||||||||||||||||
| "./routes/filesystem-mcp" | ||||||||||||||||||||
| ); | ||||||||||||||||||||
| return handleFilesystemMcpRequest(c.req.raw, ctx); | ||||||||||||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. P2: Validate Prompt for AI agents
Suggested change
|
||||||||||||||||||||
| }, | ||||||||||||||||||||
| ); | ||||||||||||||||||||
|
|
||||||||||||||||||||
| // Handle call-tool endpoint for filesystem connections | ||||||||||||||||||||
| app.all( | ||||||||||||||||||||
| "/mcp/:connectionId{.*_filesystem$}/call-tool/:toolName", | ||||||||||||||||||||
| mcpAuth, | ||||||||||||||||||||
| async (c: Context<Env>) => { | ||||||||||||||||||||
| const ctx = c.get("meshContext") as MeshContext; | ||||||||||||||||||||
| const toolName = c.req.param("toolName"); | ||||||||||||||||||||
| if (!toolName) { | ||||||||||||||||||||
| return c.json({ error: "Missing tool name" }, 400); | ||||||||||||||||||||
| } | ||||||||||||||||||||
| const args = (await c.req.json()) as Record<string, unknown>; | ||||||||||||||||||||
| const { callFilesystemTool } = await import("./routes/filesystem-mcp"); | ||||||||||||||||||||
| const result = await callFilesystemTool(toolName, args, ctx); | ||||||||||||||||||||
| if (result.isError) { | ||||||||||||||||||||
| return c.json(result.content, 500); | ||||||||||||||||||||
| } | ||||||||||||||||||||
| return c.json(result.content); | ||||||||||||||||||||
| }, | ||||||||||||||||||||
| ); | ||||||||||||||||||||
|
|
||||||||||||||||||||
| // Virtual MCP / Agent routes (must be before proxy to match /mcp/gateway and /mcp/virtual-mcp before /mcp/:connectionId) | ||||||||||||||||||||
| // /mcp/gateway/:virtualMcpId (backward compat) or /mcp/virtual-mcp/:virtualMcpId | ||||||||||||||||||||
| app.route("/mcp", virtualMcpRoutes); | ||||||||||||||||||||
|
|
||||||||||||||||||||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,306 @@ | ||
| /** | ||
| * Filesystem MCP Server | ||
| * | ||
| * An MCP server that implements the FILESYSTEM_BINDING interface | ||
| * using S3-compatible storage as the backing store. | ||
| * | ||
| * Provides inline content access for AI agents (read/write file content | ||
| * directly in tool calls), unlike OBJECT_STORAGE_BINDING which uses presigned URLs. | ||
| * | ||
| * Route: POST /mcp/filesystem | ||
| * Also handles org-scoped connection ID pattern: /mcp/{orgId}_filesystem | ||
| */ | ||
|
|
||
| import type { | ||
| FsDeleteInput, | ||
| FsDeleteOutput, | ||
| FsListInput, | ||
| FsListOutput, | ||
| FsMetadataInput, | ||
| FsMetadataOutput, | ||
| FsReadInput, | ||
| FsReadOutput, | ||
| FsWriteInput, | ||
| FsWriteOutput, | ||
| } from "@decocms/bindings/filesystem"; | ||
| import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; | ||
| import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js"; | ||
| import { Hono } from "hono"; | ||
| import { z } from "zod"; | ||
| import type { MeshContext } from "../../core/mesh-context"; | ||
| import { requireOrganization } from "../../core/mesh-context"; | ||
| import { S3Service } from "../../filesystem/s3-service"; | ||
| import { | ||
| FsDeleteInputSchema, | ||
| FsDeleteOutputSchema, | ||
| FsListInputSchema, | ||
| FsListOutputSchema, | ||
| FsMetadataInputSchema, | ||
| FsMetadataOutputSchema, | ||
| FsReadInputSchema, | ||
| FsReadOutputSchema, | ||
| FsWriteInputSchema, | ||
| FsWriteOutputSchema, | ||
| } from "../../tools/filesystem/schema"; | ||
| import { getFilesystemS3Service } from "../../filesystem/factory"; | ||
|
|
||
| // Local tool definition type | ||
| interface ToolDefinition { | ||
| name: string; | ||
| description?: string; | ||
| inputSchema: z.ZodTypeAny; | ||
| outputSchema?: z.ZodTypeAny; | ||
| handler: (args: Record<string, unknown>) => Promise<unknown>; | ||
| annotations?: { | ||
| title?: string; | ||
| readOnlyHint?: boolean; | ||
| destructiveHint?: boolean; | ||
| idempotentHint?: boolean; | ||
| openWorldHint?: boolean; | ||
| }; | ||
| } | ||
|
|
||
| // Define Hono variables type | ||
| type Variables = { | ||
| meshContext: MeshContext; | ||
| }; | ||
|
|
||
| const app = new Hono<{ Variables: Variables }>(); | ||
|
|
||
| function createFilesystemTools(s3: S3Service, orgId: string): ToolDefinition[] { | ||
| return [ | ||
| { | ||
| name: "FS_READ", | ||
| description: | ||
| "Read file content. Returns content inline for text files and small binary files (as base64). Returns an error for files exceeding the size limit.", | ||
| inputSchema: FsReadInputSchema, | ||
| outputSchema: FsReadOutputSchema, | ||
| annotations: { | ||
| title: "Read File", | ||
| readOnlyHint: true, | ||
| destructiveHint: false, | ||
| }, | ||
| handler: async (args): Promise<FsReadOutput> => { | ||
| const input = args as FsReadInput; | ||
| return s3.readFile(orgId, input.path, input.offset, input.limit); | ||
| }, | ||
| }, | ||
| { | ||
| name: "FS_WRITE", | ||
| description: | ||
| "Write content to a file. Creates the file if it doesn't exist, overwrites if it does.", | ||
| inputSchema: FsWriteInputSchema, | ||
| outputSchema: FsWriteOutputSchema, | ||
| annotations: { | ||
| title: "Write File", | ||
| readOnlyHint: false, | ||
| destructiveHint: false, | ||
| idempotentHint: true, | ||
| }, | ||
| handler: async (args): Promise<FsWriteOutput> => { | ||
| const input = args as FsWriteInput; | ||
| return s3.writeFile( | ||
| orgId, | ||
| input.path, | ||
| input.content, | ||
| input.encoding, | ||
| input.contentType, | ||
| ); | ||
| }, | ||
| }, | ||
| { | ||
| name: "FS_LIST", | ||
| description: | ||
| "List files and directories at a given path. Supports pagination and glob pattern filtering on file names.", | ||
| inputSchema: FsListInputSchema, | ||
| outputSchema: FsListOutputSchema, | ||
| annotations: { | ||
| title: "List Files", | ||
| readOnlyHint: true, | ||
| destructiveHint: false, | ||
| }, | ||
| handler: async (args): Promise<FsListOutput> => { | ||
| const input = args as FsListInput; | ||
| return s3.listFiles(orgId, input); | ||
| }, | ||
| }, | ||
| { | ||
| name: "FS_DELETE", | ||
| description: "Delete a single file.", | ||
| inputSchema: FsDeleteInputSchema, | ||
| outputSchema: FsDeleteOutputSchema, | ||
| annotations: { | ||
| title: "Delete File", | ||
| readOnlyHint: false, | ||
| destructiveHint: true, | ||
| idempotentHint: true, | ||
| }, | ||
| handler: async (args): Promise<FsDeleteOutput> => { | ||
| const input = args as FsDeleteInput; | ||
| return s3.deleteFile(orgId, input.path); | ||
| }, | ||
| }, | ||
| { | ||
| name: "FS_METADATA", | ||
| description: | ||
| "Get file metadata including size, content type, last modified time, and ETag.", | ||
| inputSchema: FsMetadataInputSchema, | ||
| outputSchema: FsMetadataOutputSchema, | ||
| annotations: { | ||
| title: "File Metadata", | ||
| readOnlyHint: true, | ||
| destructiveHint: false, | ||
| }, | ||
| handler: async (args): Promise<FsMetadataOutput> => { | ||
| const input = args as FsMetadataInput; | ||
| return s3.getMetadata(orgId, input.path); | ||
| }, | ||
| }, | ||
| ]; | ||
| } | ||
|
|
||
| /** | ||
| * Handle a filesystem MCP request with a given context | ||
| */ | ||
| export async function handleFilesystemMcpRequest( | ||
| req: Request, | ||
| ctx: MeshContext, | ||
| ): Promise<Response> { | ||
| const org = requireOrganization(ctx); | ||
| const s3 = getFilesystemS3Service(); | ||
|
|
||
| if (!s3) { | ||
| return new Response( | ||
| JSON.stringify({ | ||
| error: "Filesystem not configured. Set S3_* environment variables.", | ||
| }), | ||
| { status: 503, headers: { "Content-Type": "application/json" } }, | ||
| ); | ||
| } | ||
|
|
||
| const tools = createFilesystemTools(s3, org.id); | ||
|
|
||
| const server = new McpServer( | ||
| { name: "filesystem-mcp", version: "1.0.0" }, | ||
| { capabilities: { tools: {} } }, | ||
| ); | ||
|
|
||
| for (const tool of tools) { | ||
| const inputShape = | ||
| "shape" in tool.inputSchema | ||
| ? (tool.inputSchema.shape as z.ZodRawShape) | ||
| : z.object({}).shape; | ||
| const outputShape = | ||
| tool.outputSchema && "shape" in tool.outputSchema | ||
| ? (tool.outputSchema.shape as z.ZodRawShape) | ||
| : z.object({}).shape; | ||
|
|
||
| server.registerTool( | ||
| tool.name, | ||
| { | ||
| description: tool.description ?? "", | ||
| inputSchema: inputShape, | ||
| outputSchema: outputShape, | ||
| annotations: tool.annotations, | ||
| }, | ||
| async (args) => { | ||
| try { | ||
| const result = await tool.handler(args); | ||
| return { | ||
| content: [{ type: "text" as const, text: JSON.stringify(result) }], | ||
| structuredContent: result as { [x: string]: unknown }, | ||
| }; | ||
| } catch (error) { | ||
| const err = error as Error; | ||
| return { | ||
| content: [{ type: "text" as const, text: `Error: ${err.message}` }], | ||
| isError: true, | ||
| }; | ||
| } | ||
| }, | ||
| ); | ||
| } | ||
|
|
||
| const transport = new WebStandardStreamableHTTPServerTransport({ | ||
| enableJsonResponse: | ||
| req.headers.get("Accept")?.includes("application/json") ?? false, | ||
| }); | ||
| await server.connect(transport); | ||
| return transport.handleRequest(req); | ||
| } | ||
|
|
||
| /** | ||
| * Call a filesystem tool directly | ||
| */ | ||
| export async function callFilesystemTool( | ||
| toolName: string, | ||
| args: Record<string, unknown>, | ||
| ctx: MeshContext, | ||
| ): Promise<{ content: unknown; isError?: boolean }> { | ||
| const org = requireOrganization(ctx); | ||
| const s3 = getFilesystemS3Service(); | ||
|
|
||
| if (!s3) { | ||
| return { | ||
| content: [ | ||
| { | ||
| type: "text", | ||
| text: "Filesystem not configured. Set S3_* environment variables.", | ||
| }, | ||
| ], | ||
| isError: true, | ||
| }; | ||
| } | ||
|
|
||
| const tools = createFilesystemTools(s3, org.id); | ||
| const tool = tools.find((t) => t.name === toolName); | ||
|
|
||
| if (!tool) { | ||
| return { | ||
| content: [{ type: "text", text: `Tool not found: ${toolName}` }], | ||
| isError: true, | ||
| }; | ||
| } | ||
|
|
||
| const parsed = tool.inputSchema.safeParse(args); | ||
| if (!parsed.success) { | ||
| return { | ||
| content: [ | ||
| { | ||
| type: "text", | ||
| text: `Invalid input: ${parsed.error.message}`, | ||
| }, | ||
| ], | ||
| isError: true, | ||
| }; | ||
| } | ||
|
|
||
| try { | ||
| const result = await tool.handler(parsed.data as Record<string, unknown>); | ||
| return { | ||
| content: [{ type: "text", text: JSON.stringify(result) }], | ||
| }; | ||
| } catch (error) { | ||
| return { | ||
| content: [ | ||
| { | ||
| type: "text", | ||
| text: error instanceof Error ? error.message : String(error), | ||
| }, | ||
| ], | ||
| isError: true, | ||
| }; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Filesystem MCP endpoint | ||
| * | ||
| * Route: POST /mcp/filesystem | ||
| */ | ||
| app.all("/", async (c) => { | ||
| const ctx = c.get("meshContext"); | ||
| return handleFilesystemMcpRequest(c.req.raw, ctx); | ||
| }); | ||
|
|
||
| export default app; |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
P1: Add connection/tool authorization before executing filesystem MCP requests; these routes currently bypass the normal AccessControl checks and expose org file operations to any authenticated caller.
Prompt for AI agents