From e5e3437421af709551d8177fcea376762551c636 Mon Sep 17 00:00:00 2001 From: Alem Tuzlak Date: Mon, 2 Feb 2026 20:10:48 +0100 Subject: [PATCH 1/3] feat: multiple modalities from the client --- docs/guides/multimodal-content.md | 160 ++++++++ examples/ts-react-chat/src/routes/index.tsx | 239 +++++++++-- .../ai-anthropic/src/adapters/text.ts | 13 +- .../typescript/ai-client/src/chat-client.ts | 93 ++++- packages/typescript/ai-client/src/events.ts | 27 +- packages/typescript/ai-client/src/index.ts | 2 + packages/typescript/ai-client/src/types.ts | 38 ++ .../ai-client/tests/chat-client.test.ts | 271 +++++++++++- .../ai-devtools/src/store/ai-context.tsx | 97 +++-- .../typescript/ai-gemini/src/adapters/text.ts | 15 +- .../typescript/ai-grok/src/adapters/text.ts | 8 +- .../typescript/ai-openai/src/adapters/text.ts | 8 +- .../ai-openrouter/src/adapters/text.ts | 8 +- .../ai/src/activities/chat/messages.ts | 52 ++- .../src/activities/chat/stream/processor.ts | 41 +- packages/typescript/ai/src/index.ts | 3 + packages/typescript/ai/src/types.ts | 4 + packages/typescript/ai/src/utils.ts | 41 ++ .../ai/tests/message-converters.test.ts | 387 ++++++++++++++++++ .../adapters/fixtures/jpgfixture.jpg | Bin 0 -> 58085 bytes .../adapters/fixtures/pngfixture.png | Bin 0 -> 1924358 bytes .../adapters/src/adapters/index.ts | 2 +- .../smoke-tests/adapters/src/tests/index.ts | 42 +- .../src/tests/mmi-multimodal-image.ts | 195 +++++++++ .../src/tests/mms-multimodal-structured.ts | 250 +++++++++++ 25 files changed, 1892 insertions(+), 104 deletions(-) create mode 100644 packages/typescript/ai/src/utils.ts create mode 100644 packages/typescript/ai/tests/message-converters.test.ts create mode 100644 packages/typescript/smoke-tests/adapters/fixtures/jpgfixture.jpg create mode 100644 packages/typescript/smoke-tests/adapters/fixtures/pngfixture.png create mode 100644 packages/typescript/smoke-tests/adapters/src/tests/mmi-multimodal-image.ts create mode 100644 packages/typescript/smoke-tests/adapters/src/tests/mms-multimodal-structured.ts diff --git a/docs/guides/multimodal-content.md b/docs/guides/multimodal-content.md index 65100420d..946366923 100644 --- a/docs/guides/multimodal-content.md +++ b/docs/guides/multimodal-content.md @@ -315,3 +315,163 @@ const stream = chat({ 3. **Check model support**: Not all models support all modalities. Verify the model you're using supports the content types you want to send. 4. **Handle errors gracefully**: When a model doesn't support a particular modality, it may throw an error. Handle these cases in your application. + +## Client-Side Multimodal Messages + +When using the `ChatClient` from `@tanstack/ai-client`, you can send multimodal messages directly from your UI using the `sendMessage` method. + +### Basic Usage + +The `sendMessage` method accepts either a simple string or a `MultimodalContent` object: + +```typescript +import { ChatClient, fetchServerSentEvents } from '@tanstack/ai-client' + +const client = new ChatClient({ + connection: fetchServerSentEvents('/api/chat'), +}) + +// Simple text message +await client.sendMessage('Hello!') + +// Multimodal message with image +await client.sendMessage({ + content: [ + { type: 'text', content: 'What is in this image?' }, + { + type: 'image', + source: { type: 'url', value: 'https://example.com/photo.jpg' } + } + ] +}) +``` + +### Custom Message ID + +You can provide a custom ID for the message: + +```typescript +await client.sendMessage({ + content: 'Hello!', + id: 'custom-message-id-123' +}) +``` + +### Per-Message Body Parameters + +The second parameter allows you to pass additional body parameters for that specific request. These are shallow-merged with the client's base body configuration, with per-message parameters taking priority: + +```typescript +const client = new ChatClient({ + connection: fetchServerSentEvents('/api/chat'), + body: { model: 'gpt-5' }, // Base body params +}) + +// Override model for this specific message +await client.sendMessage('Analyze this complex problem', { + model: 'gpt-5', + temperature: 0.2, +}) + + +``` + +### React Example + +Here's how to use multimodal messages in a React component: + +```tsx +import { useChat } from '@tanstack/ai-react' +import { fetchServerSentEvents } from '@tanstack/ai-client' +import { useState } from 'react' + +function ChatWithImages() { + const [imageUrl, setImageUrl] = useState('') + const { sendMessage, messages } = useChat({ + connection: fetchServerSentEvents('/api/chat'), + }) + + const handleSendWithImage = () => { + if (imageUrl) { + sendMessage({ + content: [ + { type: 'text', content: 'What do you see in this image?' }, + { type: 'image', source: { type: 'url', value: imageUrl } } + ] + }) + } + } + + return ( +
+ setImageUrl(e.target.value)} + /> + +
+ ) +} +``` + +### File Upload Example + +Here's how to handle file uploads and send them as multimodal content: + +```tsx +import { useChat } from '@tanstack/ai-react' +import { fetchServerSentEvents } from '@tanstack/ai-client' + +function ChatWithFileUpload() { + const { sendMessage } = useChat({ + connection: fetchServerSentEvents('/api/chat'), + }) + + const handleFileUpload = async (file: File) => { + // Convert file to base64 + const base64 = await new Promise((resolve) => { + const reader = new FileReader() + reader.onload = () => { + const result = reader.result as string + // Remove data URL prefix (e.g., "data:image/png;base64,") + resolve(result.split(',')[1]) + } + reader.readAsDataURL(file) + }) + + // Determine content type based on file type + const type = file.type.startsWith('image/') + ? 'image' + : file.type.startsWith('audio/') + ? 'audio' + : file.type.startsWith('video/') + ? 'video' + : 'document' + + await sendMessage({ + content: [ + { type: 'text', content: `Please analyze this ${type}` }, + { + type, + source: { type: 'data', value: base64 }, + metadata: { mimeType: file.type } + } + ] + }) + } + + return ( + { + const file = e.target.files?.[0] + if (file) handleFileUpload(file) + }} + /> + ) +} +``` + diff --git a/examples/ts-react-chat/src/routes/index.tsx b/examples/ts-react-chat/src/routes/index.tsx index c9436c7a9..8e75f25c4 100644 --- a/examples/ts-react-chat/src/routes/index.tsx +++ b/examples/ts-react-chat/src/routes/index.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useRef, useState } from 'react' import { createFileRoute } from '@tanstack/react-router' -import { Send, Square } from 'lucide-react' +import { ImagePlus, Send, Square, X } from 'lucide-react' import ReactMarkdown from 'react-markdown' import rehypeRaw from 'rehype-raw' import rehypeSanitize from 'rehype-sanitize' @@ -10,6 +10,7 @@ import { fetchServerSentEvents, useChat } from '@tanstack/ai-react' import { clientTools } from '@tanstack/ai-client' import { ThinkingPart } from '@tanstack/ai-react-ui' import type { UIMessage } from '@tanstack/ai-react' +import type { ContentPart } from '@tanstack/ai' import type { ModelOption } from '@/lib/model-selection' import GuitarRecommendation from '@/components/example-GuitarRecommendation' import { @@ -20,6 +21,13 @@ import { } from '@/lib/guitar-tools' import { DEFAULT_MODEL_OPTION, MODEL_OPTIONS } from '@/lib/model-selection' +/** + * Generate a random message ID + */ +function generateMessageId(): string { + return `msg-${Date.now()}-${Math.random().toString(36).substring(2, 9)}` +} + const getPersonalGuitarPreferenceToolClient = getPersonalGuitarPreferenceToolDef.client(() => ({ preference: 'acoustic' })) @@ -148,6 +156,23 @@ function Messages({ ) } + // Render image parts + if (part.type === 'image') { + const imageUrl = + part.source.type === 'url' + ? part.source.value + : `data:image/png;base64,${part.source.value}` + return ( +
+ Attached image +
+ ) + } + // Approval UI if ( part.type === 'tool-call' && @@ -226,6 +251,10 @@ function Messages({ function ChatPage() { const [selectedModel, setSelectedModel] = useState(DEFAULT_MODEL_OPTION) + const [attachedImages, setAttachedImages] = useState< + Array<{ id: string; base64: string; mimeType: string; preview: string }> + >([]) + const fileInputRef = useRef(null) const body = useMemo( () => ({ @@ -243,6 +272,104 @@ function ChatPage() { }) const [input, setInput] = useState('') + /** + * Handle file selection for image attachment + */ + const handleFileSelect = async (e: React.ChangeEvent) => { + const files = e.target.files + if (!files || files.length === 0) return + + const newImages: Array<{ + id: string + base64: string + mimeType: string + preview: string + }> = [] + + for (const file of Array.from(files)) { + if (!file.type.startsWith('image/')) continue + + const base64 = await new Promise((resolve) => { + const reader = new FileReader() + reader.onload = () => { + const result = reader.result as string + // Remove data URL prefix (e.g., "data:image/png;base64,") + resolve(result.split(',')[1]) + } + reader.readAsDataURL(file) + }) + + const preview = URL.createObjectURL(file) + newImages.push({ + id: generateMessageId(), + base64, + mimeType: file.type, // Capture the actual mime type + preview, + }) + } + + setAttachedImages((prev) => [...prev, ...newImages]) + + // Reset the file input + if (fileInputRef.current) { + fileInputRef.current.value = '' + } + } + + /** + * Remove an attached image + */ + const removeImage = (id: string) => { + setAttachedImages((prev) => { + const image = prev.find((img) => img.id === id) + if (image) { + URL.revokeObjectURL(image.preview) + } + return prev.filter((img) => img.id !== id) + }) + } + + /** + * Send message with optional image attachments + */ + const handleSendMessage = () => { + if (!input.trim() && attachedImages.length === 0) return + + if (attachedImages.length > 0) { + // Build multimodal content array + const contentParts: Array = [] + + // Add text if present + if (input.trim()) { + contentParts.push({ type: 'text', content: input.trim() }) + } + + // Add images with mime type metadata + for (const img of attachedImages) { + contentParts.push({ + type: 'image', + source: { type: 'data', value: img.base64 }, + metadata: { mediaType: img.mimeType, mimeType: img.mimeType }, + }) + } + + // Send with custom message ID + sendMessage({ + content: contentParts, + id: generateMessageId(), + }) + + // Clean up image previews + attachedImages.forEach((img) => URL.revokeObjectURL(img.preview)) + setAttachedImages([]) + } else { + // Simple text message + sendMessage(input.trim()) + } + + setInput('') + } + return (
{/* Chat */} @@ -295,41 +422,89 @@ function ChatPage() {
)} -
-