AI TOOLKIT
Get started
Recipes/Next.js

Stream Text with Image Prompt

Learn how to stream text with an image prompt using the AI TOOLKIT and Next.js

2 min readnextstreamingmultimodalView source

Vision models such as GPT-4o can process both text and images. In this example, we will show you how to send an image URL along with the user's message to the model with useChat.

Run it locally

$ npm install ai

Using Image URLs

### Server

The server route uses convertToModelMessages to handle the conversion from UIMessages to model messages, which automatically handles multimodal content including images.

import { streamText } from 'ai-toolkit';

export const maxDuration = 60;

export async function POST(req: Request) {

const { messages } = await req.json();

// Call the language model

const result = streamText({

model: 'openai/gpt-4.1',

messages: await convertToModelMessages(messages),

});

// Respond with the stream

return result.toUIMessageStreamResponse();

}

### Client

On the client side, we use the new useChat hook and send multimodal messages using the parts array.

'use client';

import { useChat } from '@ai-toolkit/react';

import { DefaultChatTransport } from 'ai-toolkit';

import { useState } from 'react';

// Allow streaming responses up to 30 seconds

export const maxDuration = 30;

export default function Chat() {

const [input, setInput] = useState('');

const [imageUrl, setImageUrl] = useState(

'https://science.nasa.gov/wp-content/uploads/2023/09/web-first-images-release.png',

);

const { messages, sendMessage } = useChat();

const handleSubmit = async (event: React.FormEvent<HTMLFormElement>) => {

event.preventDefault();

sendMessage({

role: 'user',

parts: [

// check if imageUrl is defined, if so, add it to the message

...(imageUrl.trim().length > 0

? [

{

type: 'file' as const,

mediaType: 'image/png',

url: imageUrl,

},

]

: []),

{ type: 'text' as const, text: input },

],

});

setInput('');

setImageUrl('');

};

return (

<div>

<div>

{messages.map(m => (

<div key={m.id}>

<span>{m.role === 'user' ? 'User: ' : 'AI: '}</span>

<div>

{m.parts.map((part, i) => {

switch (part.type) {

case 'text':

return part.text;

case 'file':

return (

<img

key={(part.filename || 'image') + i}

src={part.url}

alt={part.filename ?? 'image'}

/>

);

default:

return null;

}

})}

</div>

</div>

))}

</div>

<form onSubmit={handleSubmit}>

<div>

<label htmlFor="image-url">Image URL:</label>

<input

id="image-url"

value={imageUrl}

placeholder="Enter image URL..."

onChange={e => setImageUrl(e.currentTarget.value)}

/>

</div>

<div>

<label htmlFor="image-description">Prompt:</label>

<input

id="image-description"

value={input}

placeholder="What does the image show..."

onChange={e => setInput(e.currentTarget.value)}

/>

</div>

<button type="submit">Send Message</button>

</form>

</div>

);

}

app/api/chat/route.ts
tsx
  1. 1import { streamText } from 'ai-toolkit';
  2. 2export const maxDuration = 60;
  3. 3export async function POST(req: Request) {
  4. 4 const { messages } = await req.json();
  5. 5 // Call the language model
  6. 6 const result = streamText({
  7. 7 model: 'openai/gpt-4.1',
  8. 8 messages: await convertToModelMessages(messages),
  9. 9 });
  10. 10 // Respond with the stream
  11. 11 return result.toUIMessageStreamResponse();
  12. 12}
app/page.tsx
typescript
  1. 1'use client';
  2. 2import { useChat } from '@ai-toolkit/react';
  3. 3import { DefaultChatTransport } from 'ai-toolkit';
  4. 4import { useState } from 'react';
  5. 5// Allow streaming responses up to 30 seconds
  6. 6export const maxDuration = 30;
  7. 7export default function Chat() {
  8. 8 const [input, setInput] = useState('');
  9. 9 const [imageUrl, setImageUrl] = useState(
  10. 10 'https://science.nasa.gov/wp-content/uploads/2023/09/web-first-images-release.png',
  11. 11 );
  12. 12 const { messages, sendMessage } = useChat();
  13. 13 const handleSubmit = async (event: React.FormEvent<HTMLFormElement>) => {
  14. 14 event.preventDefault();
  15. 15 sendMessage({
  16. 16 role: 'user',
  17. 17 parts: [
  18. 18 // check if imageUrl is defined, if so, add it to the message
  19. 19 ...(imageUrl.trim().length > 0
  20. 20 ? [
  21. 21 {
  22. 22 type: 'file' as const,
  23. 23 mediaType: 'image/png',
  24. 24 url: imageUrl,
  25. 25 },
  26. 26 ]
  27. 27 : []),
  28. 28 { type: 'text' as const, text: input },
  29. 29 ],
  30. 30 });
  31. 31 setInput('');
  32. 32 setImageUrl('');
  33. 33 };
  34. 34 return (
  35. 35 <div>
  36. 36 <div>
  37. 37 {messages.map(m => (
  38. 38 <div key={m.id}>
  39. 39 <span>{m.role === 'user' ? 'User: ' : 'AI: '}</span>
  40. 40 <div>
  41. 41 {m.parts.map((part, i) => {
  42. 42 switch (part.type) {
  43. 43 case 'text':
  44. 44 return part.text;
  45. 45 case 'file':
  46. 46 return (
  47. 47 <img
  48. 48 key={(part.filename || 'image') + i}
  49. 49 src={part.url}
  50. 50 alt={part.filename ?? 'image'}
  51. 51 />
  52. 52 );
  53. 53 default:
  54. 54 return null;
  55. 55 }
  56. 56 })}
  57. 57 </div>
  58. 58 </div>
  59. 59 ))}
  60. 60 </div>
  61. 61 <form onSubmit={handleSubmit}>
  62. 62 <div>
  63. 63 <label htmlFor="image-url">Image URL:</label>
  64. 64 <input
  65. 65 id="image-url"
  66. 66 value={imageUrl}
  67. 67 placeholder="Enter image URL..."
  68. 68 onChange={e => setImageUrl(e.currentTarget.value)}
  69. 69 />
  70. 70 </div>
  71. 71 <div>
  72. 72 <label htmlFor="image-description">Prompt:</label>
  73. 73 <input
  74. 74 id="image-description"
  75. 75 value={input}
  76. 76 placeholder="What does the image show..."
  77. 77 onChange={e => setInput(e.currentTarget.value)}
  78. 78 />
  79. 79 </div>
  80. 80 <button type="submit">Send Message</button>
  81. 81 </form>
  82. 82 </div>
  83. 83 );
  84. 84}
Browse all recipes