Streaming Responses Example

Build a real-time streaming chat application.

What You'll Build

A chat application that streams AI responses in real-time, creating a more responsive user experience. Responses appear word-by-word as they're generated.

Node.js CLI Streaming

streaming-cli.ts

import OpenAI from 'openai';
import * as readline from 'readline';

const client = new OpenAI({
  baseURL: 'https://superagentstack.orionixtech.com/api/v1',
  apiKey: process.env.OPENROUTER_KEY!,
  defaultHeaders: {
    'superAgentKey': process.env.SUPER_AGENT_KEY!,
  },
});

async function streamChat(userMessage: string) {
  try {
    const stream = await client.chat.completions.create({
      model: 'anthropic/claude-3-sonnet',
      messages: [{ role: 'user', content: userMessage }],
      stream: true,
      sessionId: `user-${Date.now()}`,
    });

    process.stdout.write('AI: ');

    for await (const chunk of stream) {
      const content = chunk.choices[0]?.delta?.content || '';
      process.stdout.write(content);
    }

    process.stdout.write('\n\n');
  } catch (error: any) {
    console.error('\nError:', error.message);
  }
}

async function main() {
  const rl = readline.createInterface({
    input: process.stdin,
    output: process.stdout,
  });

  console.log('Streaming chat started! Type "exit" to quit.\n');

  const askQuestion = () => {
    rl.question('You: ', async (input) => {
      if (input.toLowerCase() === 'exit') {
        console.log('Goodbye!');
        rl.close();
        return;
      }

      await streamChat(input);
      askQuestion();
    });
  };

  askQuestion();
}

main();

Next.js App Router (Complete Example)

API Route

app/api/chat/route.ts

import OpenAI from 'openai';
import { OpenAIStream, StreamingTextResponse } from 'ai';

const client = new OpenAI({
  baseURL: 'https://superagentstack.orionixtech.com/api/v1',
  apiKey: process.env.OPENROUTER_KEY!,
  defaultHeaders: {
    'superAgentKey': process.env.SUPER_AGENT_KEY!,
  },
});

export async function POST(req: Request) {
  const { messages, sessionId } = await req.json();

  const response = await client.chat.completions.create({
    model: 'anthropic/claude-3-sonnet',
    messages,
    stream: true,
    sessionId,
    saveToMemory: true,
  });

  const stream = OpenAIStream(response);
  return new StreamingTextResponse(stream);
}

Chat Component

app/chat/page.tsx

'use client';

import { useChat } from 'ai/react';
import { useEffect, useRef } from 'react';

export default function ChatPage() {
  const { messages, input, handleInputChange, handleSubmit, isLoading } = useChat({
    api: '/api/chat',
    body: {
      sessionId: `user-${Date.now()}`,
    },
  });

  const messagesEndRef = useRef<HTMLDivElement>(null);

  useEffect(() => {
    messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
  }, [messages]);

  return (
    <div className="flex flex-col h-screen max-w-2xl mx-auto p-4">
      <h1 className="text-2xl font-bold mb-4">Streaming Chat</h1>
      
      {/* Messages */}
      <div className="flex-1 overflow-y-auto mb-4 space-y-4">
        {messages.map((m) => (
          <div
            key={m.id}
            className={`p-4 rounded-lg ${
              m.role === 'user'
                ? 'bg-blue-100 ml-auto max-w-[80%]'
                : 'bg-gray-100 mr-auto max-w-[80%]'
            }`}
          >
            <div className="font-semibold mb-1">
              {m.role === 'user' ? 'You' : 'AI'}
            </div>
            <div className="whitespace-pre-wrap">{m.content}</div>
          </div>
        ))}
        {isLoading && (
          <div className="bg-gray-100 p-4 rounded-lg mr-auto max-w-[80%]">
            <div className="font-semibold mb-1">AI</div>
            <div className="animate-pulse">Thinking...</div>
          </div>
        )}
        <div ref={messagesEndRef} />
      </div>

      {/* Input */}
      <form onSubmit={handleSubmit} className="flex gap-2">
        <input
          value={input}
          onChange={handleInputChange}
          placeholder="Type your message..."
          className="flex-1 p-3 border rounded-lg focus:outline-none focus:ring-2 focus:ring-blue-500"
          disabled={isLoading}
        />
        <button
          type="submit"
          disabled={isLoading}
          className="px-6 py-3 bg-blue-500 text-white rounded-lg hover:bg-blue-600 disabled:bg-gray-300"
        >
          Send
        </button>
      </form>
    </div>
  );
}

Install Dependencies

bash

npm install ai openai

Python with FastAPI

main.py

from fastapi import FastAPI
from fastapi.responses import StreamingResponse
from openai import OpenAI
import os

app = FastAPI()

client = OpenAI(
    base_url="https://superagentstack.orionixtech.com/api/v1",
    api_key=os.environ.get("OPENROUTER_KEY"),
    default_headers={
        "superAgentKey": os.environ.get("SUPER_AGENT_KEY"),
    }
)

@app.post("/chat")
async def chat(message: str, session_id: str):
    def generate():
        stream = client.chat.completions.create(
            model="anthropic/claude-3-sonnet",
            messages=[{"role": "user", "content": message}],
            stream=True,
            session_id=session_id,
            save_to_memory=True,
        )
        
        for chunk in stream:
            if chunk.choices[0].delta.content:
                yield chunk.choices[0].delta.content
    
    return StreamingResponse(generate(), media_type="text/plain")

if __name__ == "__main__":
    import uvicorn
    uvicorn.run(app, host="0.0.0.0", port=8000)

React Component (Without Vercel AI SDK)

StreamingChat.tsx

import { useState, useRef, useEffect } from 'react';

export default function StreamingChat() {
  const [messages, setMessages] = useState<Array<{ role: string; content: string }>>([]);
  const [input, setInput] = useState('');
  const [isStreaming, setIsStreaming] = useState(false);
  const messagesEndRef = useRef<HTMLDivElement>(null);

  useEffect(() => {
    messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' });
  }, [messages]);

  async function handleSubmit(e: React.FormEvent) {
    e.preventDefault();
    if (!input.trim() || isStreaming) return;

    const userMessage = input;
    setInput('');
    setMessages(prev => [...prev, { role: 'user', content: userMessage }]);
    setIsStreaming(true);

    try {
      const response = await fetch('/api/chat', {
        method: 'POST',
        headers: { 'Content-Type': 'application/json' },
        body: JSON.stringify({
          message: userMessage,
          sessionId: `user-${Date.now()}`,
        }),
      });

      if (!response.body) throw new Error('No response body');

      const reader = response.body.getReader();
      const decoder = new TextDecoder();
      let assistantMessage = '';

      // Add empty assistant message
      setMessages(prev => [...prev, { role: 'assistant', content: '' }]);

      while (true) {
        const { done, value } = await reader.read();
        if (done) break;

        const chunk = decoder.decode(value);
        assistantMessage += chunk;

        // Update the last message
        setMessages(prev => {
          const newMessages = [...prev];
          newMessages[newMessages.length - 1].content = assistantMessage;
          return newMessages;
        });
      }
    } catch (error) {
      console.error('Error:', error);
      setMessages(prev => [
        ...prev,
        { role: 'assistant', content: 'Error: Failed to get response' },
      ]);
    } finally {
      setIsStreaming(false);
    }
  }

  return (
    <div className="flex flex-col h-screen max-w-2xl mx-auto p-4">
      <h1 className="text-2xl font-bold mb-4">Streaming Chat</h1>
      
      <div className="flex-1 overflow-y-auto mb-4 space-y-4">
        {messages.map((m, i) => (
          <div
            key={i}
            className={`p-4 rounded-lg ${
              m.role === 'user' ? 'bg-blue-100 ml-auto' : 'bg-gray-100'
            } max-w-[80%]`}
          >
            <div className="font-semibold mb-1">
              {m.role === 'user' ? 'You' : 'AI'}
            </div>
            <div className="whitespace-pre-wrap">{m.content}</div>
          </div>
        ))}
        <div ref={messagesEndRef} />
      </div>

      <form onSubmit={handleSubmit} className="flex gap-2">
        <input
          value={input}
          onChange={(e) => setInput(e.target.value)}
          placeholder="Type your message..."
          className="flex-1 p-3 border rounded-lg"
          disabled={isStreaming}
        />
        <button
          type="submit"
          disabled={isStreaming}
          className="px-6 py-3 bg-blue-500 text-white rounded-lg"
        >
          {isStreaming ? 'Sending...' : 'Send'}
        </button>
      </form>
    </div>
  );
}

Error Handling in Streams

streaming-with-errors.ts

async function streamWithErrorHandling(userMessage: string) {
  const controller = new AbortController();
  
  // Timeout after 30 seconds
  const timeout = setTimeout(() => controller.abort(), 30000);

  try {
    const stream = await client.chat.completions.create({
      model: 'anthropic/claude-3-sonnet',
      messages: [{ role: 'user', content: userMessage }],
      stream: true,
    }, {
      signal: controller.signal,
    });

    for await (const chunk of stream) {
      const content = chunk.choices[0]?.delta?.content || '';
      process.stdout.write(content);
    }

    clearTimeout(timeout);
  } catch (error: any) {
    clearTimeout(timeout);
    
    if (error.name === 'AbortError') {
      console.error('\nStream timeout - request took too long');
    } else if (error.status === 429) {
      console.error('\nRate limit exceeded - please wait');
    } else {
      console.error('\nStream error:', error.message);
    }
  }
}

Performance Tips

Buffer small chunks: Accumulate chunks before updating UI to reduce re-renders
Use debouncing: Update UI every 50-100ms instead of every chunk
Implement timeouts: Cancel streams that take too long
Show loading states: Display indicators while waiting for first chunk
Handle reconnection: Retry failed streams automatically

Buffered Updates

buffered-streaming.ts

async function streamWithBuffering(userMessage: string) {
  let buffer = '';
  let lastUpdate = Date.now();
  const UPDATE_INTERVAL = 100; // Update every 100ms

  const stream = await client.chat.completions.create({
    model: 'anthropic/claude-3-sonnet',
    messages: [{ role: 'user', content: userMessage }],
    stream: true,
  });

  for await (const chunk of stream) {
    const content = chunk.choices[0]?.delta?.content || '';
    buffer += content;

    // Update UI only every 100ms
    const now = Date.now();
    if (now - lastUpdate >= UPDATE_INTERVAL) {
      process.stdout.write(buffer);
      buffer = '';
      lastUpdate = now;
    }
  }

  // Write remaining buffer
  if (buffer) {
    process.stdout.write(buffer);
  }
}

Best Practice

Always implement proper error handling and timeouts for streaming requests. Network issues can cause streams to hang indefinitely.