Stream Text with Chat Prompt
Learn how to stream text with chat prompt using the AI TOOLKIT and React Server Components.
Chat completion can sometimes take a long time to finish, especially when the response is big. In such cases, it is useful to stream the chat completion to the client in real-time. This allows the client to display the new message as it is being generated by the model, rather than have users wait for it to finish. stream history={[ { role: 'User', content: 'How is it going?' }, { role: 'Assistant', content: 'All good, how may I help you?' }, ]} inputMessage={{ role: 'User', content: 'Why is the sky blue?' }} outputMessage={{ role: 'Assistant', content: 'The sky is blue because of rayleigh scattering.', }} />
Run it locally
$ npm install aiClient
Let's create a simple conversation between a user and a model, and place a button that will call continueConversation.
'use client';
import { useState } from 'react';
import { Message, continueConversation } from './actions';
import { readStreamableValue } from '@ai-toolkit/rsc';
// Allow streaming responses up to 30 seconds
export const maxDuration = 30;
export default function Home() {
const [conversation, setConversation] = useState<Message[]>([]);
const [input, setInput] = useState<string>('');
return (
<div>
<div>
{conversation.map((message, index) => (
<div key={index}>
{message.role}: {message.content}
</div>
))}
</div>
<div>
<input
type="text"
value={input}
onChange={event => {
setInput(event.target.value);
}}
/>
<button
onClick={async () => {
const { messages, newMessage } = await continueConversation([
...conversation,
{ role: 'user', content: input },
]);
let textContent = '';
for await (const delta of readStreamableValue(newMessage)) {
textContent = ${textContent}${delta};
setConversation([
...messages,
{ role: 'assistant', content: textContent },
]);
}
}}
>
Send Message
</button>
</div>
</div>
);
}
- 1'use client';
- 2import { useState } from 'react';
- 3import { Message, continueConversation } from './actions';
- 4import { readStreamableValue } from '@ai-toolkit/rsc';
- 5// Allow streaming responses up to 30 seconds
- 6export const maxDuration = 30;
- 7export default function Home() {
- 8 const [conversation, setConversation] = useState<Message[]>([]);
- 9 const [input, setInput] = useState<string>('');
- 10 return (
- 11 <div>
- 12 <div>
- 13 {conversation.map((message, index) => (
- 14 <div key={index}>
- 15 {message.role}: {message.content}
- 16 </div>
- 17 ))}
- 18 </div>
- 19 <div>
- 20 <input
- 21 type="text"
- 22 value={input}
- 23 onChange={event => {
- 24 setInput(event.target.value);
- 25 }}
- 26 />
- 27 <button
- 28 onClick={async () => {
- 29 const { messages, newMessage } = await continueConversation([
- 30 ...conversation,
- 31 { role: 'user', content: input },
- 32 ]);
- 33 let textContent = '';
- 34 for await (const delta of readStreamableValue(newMessage)) {
- 35 textContent = `${textContent}${delta}`;
- 36 setConversation([
- 37 ...messages,
- 38 { role: 'assistant', content: textContent },
- 39 ]);
- 40 }
- 41 }}
- 42 >
- 43 Send Message
- 44 </button>
- 45 </div>
- 46 </div>
- 47 );
- 48}
Server
Now, let's implement the continueConversation function that will insert the user's message into the conversation and stream back the new message.
'use server';
import { streamText } from 'ai-toolkit';
import { openai } from '@ai-toolkit/openai';
import { createStreamableValue } from '@ai-toolkit/rsc';
export interface Message {
role: 'user' | 'assistant';
content: string;
}
export async function continueConversation(history: Message[]) {
'use server';
const stream = createStreamableValue();
(async () => {
const { textStream } = streamText({
model: openai('gpt-3.5-turbo'),
system:
"You are a dude that doesn't drop character until the DVD commentary.",
messages: history,
});
for await (const text of textStream) {
stream.update(text);
}
stream.done();
})();
return {
messages: history,
newMessage: stream.value,
};
}
- 1'use server';
- 2import { streamText } from 'ai-toolkit';
- 3import { openai } from '@ai-toolkit/openai';
- 4import { createStreamableValue } from '@ai-toolkit/rsc';
- 5export interface Message {
- 6 role: 'user' | 'assistant';
- 7 content: string;
- 8}
- 9export async function continueConversation(history: Message[]) {
- 10 'use server';
- 11 const stream = createStreamableValue();
- 12 (async () => {
- 13 const { textStream } = streamText({
- 14 model: openai('gpt-3.5-turbo'),
- 15 system:
- 16 "You are a dude that doesn't drop character until the DVD commentary.",
- 17 messages: history,
- 18 });
- 19 for await (const text of textStream) {
- 20 stream.update(text);
- 21 }
- 22 stream.done();
- 23 })();
- 24 return {
- 25 messages: history,
- 26 newMessage: stream.value,
- 27 };
- 28}