Add image attachment & vision support

README.md

@@ -6,7 +6,7 @@
 
 A starter template for building AI chat agents on Cloudflare, powered by the [Agents SDK](https://developers.cloudflare.com/agents/).
 
-Uses Workers AI (no API key required), with tools for weather, timezone detection, calculations with approval, and task scheduling.
+Uses Workers AI (no API key required), with tools for weather, timezone detection, calculations with approval, task scheduling, and vision (image input).
 
 ## Quick start
 
@@ -25,6 +25,7 @@ Try these prompts to see the different features:
 - **"What timezone am I in?"** — client-side tool (browser provides the answer)
 - **"Calculate 5000 \* 3"** — approval tool (asks you before running)
 - **"Remind me in 5 minutes to take a break"** — scheduling
+- **Drop an image and ask "What's in this image?"** — vision (image understanding)
 
 ## Project structure
 
@@ -39,6 +40,7 @@ src/
 ## What's included
 
 - **AI Chat** — Streaming responses powered by Workers AI via `AIChatAgent`
+- **Image input** — Drag-and-drop, paste, or click to attach images for vision-capable models
 - **Three tool patterns** — server-side auto-execute, client-side (browser), and human-in-the-loop approval
 - **Scheduling** — one-time, delayed, and recurring (cron) tasks
 - **Reasoning display** — shows model thinking as it streams, collapses when done

package.json

@@ -28,7 +28,7 @@
     "react": "^19.2.4",
     "react-dom": "^19.2.4",
     "streamdown": "^2.2.0",
-    "workers-ai-provider": "^3.1.3",
+    "workers-ai-provider": "^3.1.4",
     "zod": "^4.3.6"
   },
   "devDependencies": {

src/app.tsx

@@ -33,9 +33,38 @@ import {
   PlusIcon,
   SignInIcon,
   XIcon,
-  WrenchIcon
+  WrenchIcon,
+  PaperclipIcon,
+  ImageIcon
 } from "@phosphor-icons/react";
 
+// ── Attachment helpers ────────────────────────────────────────────────
+
+interface Attachment {
+  id: string;
+  file: File;
+  preview: string;
+  mediaType: string;
+}
+
+function createAttachment(file: File): Attachment {
+  return {
+    id: `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
+    file,
+    preview: URL.createObjectURL(file),
+    mediaType: file.type || "application/octet-stream"
+  };
+}
+
+function fileToDataUri(file: File): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onload = () => resolve(reader.result as string);
+    reader.onerror = reject;
+    reader.readAsDataURL(file);
+  });
+}
+
 // ── Small components ──────────────────────────────────────────────────
 
 function ThemeToggle() {
@@ -194,8 +223,11 @@ function Chat() {
   const [connected, setConnected] = useState(false);
   const [input, setInput] = useState("");
   const [showDebug, setShowDebug] = useState(false);
+  const [attachments, setAttachments] = useState<Attachment[]>([]);
+  const [isDragging, setIsDragging] = useState(false);
   const messagesEndRef = useRef<HTMLDivElement>(null);
   const textareaRef = useRef<HTMLTextAreaElement>(null);
+  const fileInputRef = useRef<HTMLInputElement>(null);
   const toasts = useKumoToastManager();
   const [mcpState, setMcpState] = useState<MCPServersState>({
     prompts: [],
@@ -321,18 +353,100 @@ function Chat() {
     }
   }, [isStreaming]);
 
-  const send = useCallback(() => {
+  const addFiles = useCallback((files: FileList | File[]) => {
+    const images = Array.from(files).filter((f) => f.type.startsWith("image/"));
+    if (images.length === 0) return;
+    setAttachments((prev) => [...prev, ...images.map(createAttachment)]);
+  }, []);
+
+  const removeAttachment = useCallback((id: string) => {
+    setAttachments((prev) => {
+      const att = prev.find((a) => a.id === id);
+      if (att) URL.revokeObjectURL(att.preview);
+      return prev.filter((a) => a.id !== id);
+    });
+  }, []);
+
+  const handleDragOver = useCallback((e: React.DragEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+    if (e.dataTransfer.types.includes("Files")) setIsDragging(true);
+  }, []);
+
+  const handleDragLeave = useCallback((e: React.DragEvent) => {
+    e.preventDefault();
+    e.stopPropagation();
+    if (e.currentTarget === e.target) setIsDragging(false);
+  }, []);
+
+  const handleDrop = useCallback(
+    (e: React.DragEvent) => {
+      e.preventDefault();
+      e.stopPropagation();
+      setIsDragging(false);
+      if (e.dataTransfer.files.length > 0) addFiles(e.dataTransfer.files);
+    },
+    [addFiles]
+  );
+
+  const handlePaste = useCallback(
+    (e: React.ClipboardEvent) => {
+      const items = e.clipboardData?.items;
+      if (!items) return;
+      const files: File[] = [];
+      for (const item of items) {
+        if (item.kind === "file") {
+          const file = item.getAsFile();
+          if (file) files.push(file);
+        }
+      }
+      if (files.length > 0) {
+        e.preventDefault();
+        addFiles(files);
+      }
+    },
+    [addFiles]
+  );
+
+  const send = useCallback(async () => {
     const text = input.trim();
-    if (!text || isStreaming) return;
+    if ((!text && attachments.length === 0) || isStreaming) return;
     setInput("");
-    sendMessage({ role: "user", parts: [{ type: "text", text }] });
-    if (textareaRef.current) {
-      textareaRef.current.style.height = "auto";
+
+    const parts: Array<
+      | { type: "text"; text: string }
+      | { type: "file"; mediaType: string; url: string }
+    > = [];
+    if (text) parts.push({ type: "text", text });
+
+    for (const att of attachments) {
+      const dataUri = await fileToDataUri(att.file);
+      parts.push({ type: "file", mediaType: att.mediaType, url: dataUri });
     }
-  }, [input, isStreaming, sendMessage]);
+
+    for (const att of attachments) URL.revokeObjectURL(att.preview);
+    setAttachments([]);
+
+    sendMessage({ role: "user", parts });
+    if (textareaRef.current) textareaRef.current.style.height = "auto";
+  }, [input, attachments, isStreaming, sendMessage]);
 
   return (
-    <div className="flex flex-col h-screen bg-kumo-elevated">
+    <div
+      className="flex flex-col h-screen bg-kumo-elevated relative"
+      onDragOver={handleDragOver}
+      onDragLeave={handleDragLeave}
+      onDrop={handleDrop}
+    >
+      {isDragging && (
+        <div className="absolute inset-0 z-50 flex items-center justify-center bg-kumo-elevated/80 backdrop-blur-sm border-2 border-dashed border-kumo-brand rounded-xl m-2 pointer-events-none">
+          <div className="flex flex-col items-center gap-2 text-kumo-brand">
+            <ImageIcon size={40} />
+            <Text variant="heading3">Drop images here</Text>
+          </div>
+        </div>
+      )}
+
       {/* Header */}
       <header className="px-5 py-4 bg-kumo-base border-b border-kumo-line">
         <div className="max-w-3xl mx-auto flex items-center justify-between">
@@ -643,6 +757,28 @@ function Chat() {
                     );
                   })}
 
+                {/* Image parts */}
+                {message.parts
+                  .filter(
+                    (part): part is Extract<typeof part, { type: "file" }> =>
+                      part.type === "file" &&
+                      (part as { mediaType?: string }).mediaType?.startsWith(
+                        "image/"
+                      ) === true
+                  )
+                  .map((part, i) => (
+                    <div
+                      key={`file-${i}`}
+                      className={`flex ${isUser ? "justify-end" : "justify-start"}`}
+                    >
+                      <img
+                        src={part.url}
+                        alt="Attachment"
+                        className="max-h-64 rounded-xl border border-kumo-line object-contain"
+                      />
+                    </div>
+                  ))}
+
                 {/* Text parts */}
                 {message.parts
                   .filter((part) => part.type === "text")
@@ -691,7 +827,54 @@ function Chat() {
           }}
           className="max-w-3xl mx-auto px-5 py-4"
         >
+          <input
+            ref={fileInputRef}
+            type="file"
+            multiple
+            accept="image/*"
+            className="hidden"
+            onChange={(e) => {
+              if (e.target.files) addFiles(e.target.files);
+              e.target.value = "";
+            }}
+          />
+
+          {attachments.length > 0 && (
+            <div className="flex gap-2 mb-2 flex-wrap">
+              {attachments.map((att) => (
+                <div
+                  key={att.id}
+                  className="relative group rounded-lg border border-kumo-line bg-kumo-control overflow-hidden"
+                >
+                  <img
+                    src={att.preview}
+                    alt={att.file.name}
+                    className="h-16 w-16 object-cover"
+                  />
+                  <button
+                    type="button"
+                    onClick={() => removeAttachment(att.id)}
+                    className="absolute top-0.5 right-0.5 rounded-full bg-kumo-contrast/80 text-kumo-inverse p-0.5 opacity-0 group-hover:opacity-100 transition-opacity"
+                    aria-label={`Remove ${att.file.name}`}
+                  >
+                    <XIcon size={10} />
+                  </button>
+                </div>
+              ))}
+            </div>
+          )}
+
           <div className="flex items-end gap-3 rounded-xl border border-kumo-line bg-kumo-base p-3 shadow-sm focus-within:ring-2 focus-within:ring-kumo-ring focus-within:border-transparent transition-shadow">
+            <Button
+              type="button"
+              variant="ghost"
+              shape="square"
+              aria-label="Attach images"
+              icon={<PaperclipIcon size={18} />}
+              onClick={() => fileInputRef.current?.click()}
+              disabled={!connected || isStreaming}
+              className="mb-0.5"
+            />
             <InputArea
               ref={textareaRef}
               value={input}
@@ -707,7 +890,12 @@ function Chat() {
                 el.style.height = "auto";
                 el.style.height = `${el.scrollHeight}px`;
               }}
-              placeholder="Send a message..."
+              onPaste={handlePaste}
+              placeholder={
+                attachments.length > 0
+                  ? "Add a message or send images..."
+                  : "Send a message..."
+              }
               disabled={!connected || isStreaming}
               rows={1}
               className="flex-1 ring-0! focus:ring-0! shadow-none! bg-transparent! outline-none! resize-none max-h-40"
@@ -728,7 +916,9 @@ function Chat() {
                 variant="primary"
                 shape="square"
                 aria-label="Send message"
-                disabled={!input.trim() || !connected}
+                disabled={
+                  (!input.trim() && attachments.length === 0) || !connected
+                }
                 icon={<PaperPlaneRightIcon size={18} />}
                 className="mb-0.5"
               />

src/server.ts

@@ -7,10 +7,33 @@ import {
   convertToModelMessages,
   pruneMessages,
   tool,
-  stepCountIs
+  stepCountIs,
+  type ModelMessage
 } from "ai";
 import { z } from "zod";
 
+/**
+ * The AI SDK's downloadAssets step runs `new URL(data)` on every file
+ * part's string data. Data URIs parse as valid URLs, so it tries to
+ * HTTP-fetch them and fails. Decode to Uint8Array so the SDK treats
+ * them as inline data instead.
+ */
+function inlineDataUrls(messages: ModelMessage[]): ModelMessage[] {
+  return messages.map((msg) => {
+    if (msg.role !== "user" || typeof msg.content === "string") return msg;
+    return {
+      ...msg,
+      content: msg.content.map((part) => {
+        if (part.type !== "file" || typeof part.data !== "string") return part;
+        const match = part.data.match(/^data:([^;]+);base64,(.+)$/);
+        if (!match) return part;
+        const bytes = Uint8Array.from(atob(match[2]), (c) => c.charCodeAt(0));
+        return { ...part, data: bytes, mediaType: match[1] };
+      })
+    };
+  });
+}
+
 export class ChatAgent extends AIChatAgent<Env> {
   onStart() {
     // Configure OAuth popup behavior for MCP servers that require authentication
@@ -46,14 +69,14 @@ export class ChatAgent extends AIChatAgent<Env> {
 
     const result = streamText({
       model: workersai("@cf/moonshotai/kimi-k2.5"),
-      system: `You are a helpful assistant. You can check the weather, get the user's timezone, run calculations, and schedule tasks.
+      system: `You are a helpful assistant that can understand images. You can check the weather, get the user's timezone, run calculations, and schedule tasks. When users share images, describe what you see and answer questions about them.
 
 ${getSchedulePrompt({ date: new Date() })}
 
 If the user asks to schedule a task, use the schedule tool to schedule the task.`,
       // Prune old tool calls to save tokens on long conversations
       messages: pruneMessages({
-        messages: await convertToModelMessages(this.messages),
+        messages: inlineDataUrls(await convertToModelMessages(this.messages)),
         toolCalls: "before-last-2-messages"
       }),
       tools: {