Streaming Chat Example

Learn how to implement real-time streaming chat responses that update as the AI generates text, providing a better user experience.

Overview

Streaming allows responses to appear word-by-word as they're generated, rather than waiting for the complete response. This example shows:

• Server-Sent Events (SSE) handling
• Real-time UI updates
• Conversation state management
• Error handling for streaming

Best for: Production applications, better UX, responsive feel

Step 1: Backend Proxy with Streaming

Create a backend endpoint that streams responses:

Node.js/Express Example

// server.js
const express = require('express');
const app = express();

app.use(express.json());

app.post('/api/chatbot', async (req, res) => {
  const { message, conversation_id } = req.body;

  try {
    const response = await fetch('https://chatiq.io/api/chat', {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'Authorization': `Bearer ${process.env.CHATIQ_API_KEY}`,
      },
      body: JSON.stringify({
        message: message,
        bot_slug: process.env.CHATIQ_BOT_SLUG,
        stream: true, // Enable streaming
        conversation_id: conversation_id || null,
      }),
    });

    // Set headers for Server-Sent Events
    res.setHeader('Content-Type', 'text/event-stream');
    res.setHeader('Cache-Control', 'no-cache');
    res.setHeader('Connection', 'keep-alive');

    // Pipe the stream to the client
    const reader = response.body.getReader();
    const decoder = new TextDecoder();

    while (true) {
      const { done, value } = await reader.read();
      if (done) break;

      const chunk = decoder.decode(value, { stream: true });
      res.write(chunk);
    }

    res.end();
  } catch (error) {
    res.status(500).json({ error: 'Streaming failed' });
  }
});

app.listen(3000);

Python/Flask Example

# app.py
from flask import Flask, request, Response, stream_with_context
import os
import requests

app = Flask(__name__)

@app.route('/api/chatbot', methods=['POST'])
def chatbot():
    data = request.json
    message = data.get('message')
    conversation_id = data.get('conversation_id')

    def generate():
        response = requests.post(
            'https://chatiq.io/api/chat',
            headers={
                'Content-Type': 'application/json',
                'Authorization': f'Bearer {os.getenv("CHATIQ_API_KEY")}',
            },
            json={
                'message': message,
                'bot_slug': os.getenv('CHATIQ_BOT_SLUG'),
                'stream': True,
                'conversation_id': conversation_id,
            },
            stream=True
        )

        for chunk in response.iter_content(chunk_size=None):
            if chunk:
                yield chunk

    return Response(
        stream_with_context(generate()),
        mimetype='text/event-stream',
        headers={
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
        }
    )

if __name__ == '__main__':
    app.run(port=3000)

Step 2: Client-Side Streaming Handler

Implement the streaming handler in JavaScript:

// chat.js
let conversationId = null;

async function sendMessageStreaming(message) {
  const response = await fetch('/api/chatbot', {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
    },
    body: JSON.stringify({
      message: message,
      conversation_id: conversationId,
    }),
  });

  if (!response.body) {
    throw new Error('No response body');
  }

  const reader = response.body.getReader();
  const decoder = new TextDecoder();
  let fullResponse = '';
  let receivedConversationId = null;

  // Create a placeholder message in the UI
  const messageId = Date.now();
  addMessagePlaceholder('assistant', messageId);

  while (true) {
    const { done, value } = await reader.read();
    if (done) break;

    const chunk = decoder.decode(value, { stream: true });
    const lines = chunk.split('\n');

    for (const line of lines) {
      if (line.startsWith('data: ')) {
        const data = line.slice(6).trim();
        
        if (data === '[DONE]') {
          updateMessageContent(messageId, fullResponse);
          if (receivedConversationId) {
            conversationId = receivedConversationId;
          }
          return;
        }

        try {
          const parsed = JSON.parse(data);
          
          // Handle conversation ID
          if (parsed.conversationId) {
            receivedConversationId = parsed.conversationId;
          }

          // Handle content chunks
          const content = parsed.choices?.[0]?.delta?.content;
          if (content) {
            fullResponse += content;
            updateMessageContent(messageId, fullResponse);
          }
        } catch (e) {
          // Skip invalid JSON
          console.warn('Invalid JSON:', data);
        }
      }
    }
  }
}

// UI helper functions
function addMessagePlaceholder(role, messageId) {
  const messagesDiv = document.getElementById('messages');
  const messageDiv = document.createElement('div');
  messageDiv.id = `message-${messageId}`;
  messageDiv.className = `message ${role}`;
  messageDiv.textContent = '';
  messagesDiv.appendChild(messageDiv);
  messagesDiv.scrollTop = messagesDiv.scrollHeight;
}

function updateMessageContent(messageId, content) {
  const messageDiv = document.getElementById(`message-${messageId}`);
  if (messageDiv) {
    messageDiv.textContent = content;
    // Auto-scroll to bottom
    const messagesDiv = document.getElementById('messages');
    messagesDiv.scrollTop = messagesDiv.scrollHeight;
  }
}

Step 3: Complete HTML Example

<!DOCTYPE html>
<html lang="en">
<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <title>ChatIQ Streaming Example</title>
  <style>
    .chat-container {
      max-width: 600px;
      margin: 50px auto;
      border: 1px solid #ddd;
      border-radius: 8px;
      overflow: hidden;
      box-shadow: 0 2px 10px rgba(0,0,0,0.1);
    }
    .chat-messages {
      height: 400px;
      overflow-y: auto;
      padding: 20px;
      background: #f9f9f9;
    }
    .message {
      margin-bottom: 15px;
      padding: 10px;
      border-radius: 8px;
      white-space: pre-wrap;
      word-wrap: break-word;
    }
    .message.user {
      background: #007bff;
      color: white;
      text-align: right;
    }
    .message.assistant {
      background: white;
      border: 1px solid #ddd;
    }
    .message.assistant.streaming::after {
      content: '▋';
      animation: blink 1s infinite;
    }
    @keyframes blink {
      0%, 50% { opacity: 1; }
      51%, 100% { opacity: 0; }
    }
    .chat-input {
      display: flex;
      padding: 15px;
      background: white;
      border-top: 1px solid #ddd;
    }
    .chat-input input {
      flex: 1;
      padding: 10px;
      border: 1px solid #ddd;
      border-radius: 4px;
      margin-right: 10px;
    }
    .chat-input button {
      padding: 10px 20px;
      background: #007bff;
      color: white;
      border: none;
      border-radius: 4px;
      cursor: pointer;
    }
    .chat-input button:disabled {
      background: #ccc;
      cursor: not-allowed;
    }
  </style>
</head>
<body>
  <div class="chat-container">
    <div class="chat-messages" id="messages"></div>
    <div class="chat-input">
      <input type="text" id="messageInput" placeholder="Type your message...">
      <button id="sendButton" onclick="handleSend()">Send</button>
    </div>
  </div>

  <script>
    let conversationId = null;
    let isStreaming = false;

    async function handleSend() {
      const input = document.getElementById('messageInput');
      const button = document.getElementById('sendButton');
      const message = input.value.trim();
      
      if (!message || isStreaming) return;

      addMessage('user', message);
      input.value = '';
      button.disabled = true;
      isStreaming = true;

      try {
        await sendMessageStreaming(message);
      } catch (error) {
        addMessage('assistant', 'Sorry, something went wrong. Please try again.');
        console.error('Error:', error);
      } finally {
        button.disabled = false;
        isStreaming = false;
        input.focus();
      }
    }

    async function sendMessageStreaming(message) {
      const response = await fetch('/api/chatbot', {
        method: 'POST',
        headers: {
          'Content-Type': 'application/json',
        },
        body: JSON.stringify({
          message: message,
          conversation_id: conversationId,
        }),
      });

      if (!response.body) {
        throw new Error('No response body');
      }

      const reader = response.body.getReader();
      const decoder = new TextDecoder();
      let fullResponse = '';
      let receivedConversationId = null;

      const messageId = Date.now();
      addMessagePlaceholder('assistant', messageId);

      while (true) {
        const { done, value } = await reader.read();
        if (done) break;

        const chunk = decoder.decode(value, { stream: true });
        const lines = chunk.split('\n');

        for (const line of lines) {
          if (line.startsWith('data: ')) {
            const data = line.slice(6).trim();
            
            if (data === '[DONE]') {
              removeStreamingIndicator(messageId);
              if (receivedConversationId) {
                conversationId = receivedConversationId;
              }
              return;
            }

            try {
              const parsed = JSON.parse(data);
              
              if (parsed.conversationId) {
                receivedConversationId = parsed.conversationId;
              }

              const content = parsed.choices?.[0]?.delta?.content;
              if (content) {
                fullResponse += content;
                updateMessageContent(messageId, fullResponse);
                addStreamingIndicator(messageId);
              }
            } catch (e) {
              // Skip invalid JSON
            }
          }
        }
      }
    }

    function addMessage(role, content) {
      const messagesDiv = document.getElementById('messages');
      const messageDiv = document.createElement('div');
      messageDiv.className = `message ${role}`;
      messageDiv.textContent = content;
      messagesDiv.appendChild(messageDiv);
      messagesDiv.scrollTop = messagesDiv.scrollHeight;
    }

    function addMessagePlaceholder(role, messageId) {
      const messagesDiv = document.getElementById('messages');
      const messageDiv = document.createElement('div');
      messageDiv.id = `message-${messageId}`;
      messageDiv.className = `message ${role}`;
      messageDiv.textContent = '';
      messagesDiv.appendChild(messageDiv);
      messagesDiv.scrollTop = messagesDiv.scrollHeight;
    }

    function updateMessageContent(messageId, content) {
      const messageDiv = document.getElementById(`message-${messageId}`);
      if (messageDiv) {
        messageDiv.textContent = content;
        const messagesDiv = document.getElementById('messages');
        messagesDiv.scrollTop = messagesDiv.scrollHeight;
      }
    }

    function addStreamingIndicator(messageId) {
      const messageDiv = document.getElementById(`message-${messageId}`);
      if (messageDiv && !messageDiv.classList.contains('streaming')) {
        messageDiv.classList.add('streaming');
      }
    }

    function removeStreamingIndicator(messageId) {
      const messageDiv = document.getElementById(`message-${messageId}`);
      if (messageDiv) {
        messageDiv.classList.remove('streaming');
      }
    }

    // Enter key support
    document.getElementById('messageInput').addEventListener('keypress', (e) => {
      if (e.key === 'Enter' && !isStreaming) {
        handleSend();
      }
    });
  </script>
</body>
</html>

Step 4: Error Handling

Add robust error handling for network issues:

async function sendMessageStreaming(message) {
  try {
    const response = await fetch('/api/chatbot', {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
      },
      body: JSON.stringify({
        message: message,
        conversation_id: conversationId,
      }),
    });

    if (!response.ok) {
      throw new Error(`HTTP error! status: ${response.status}`);
    }

    if (!response.body) {
      throw new Error('No response body');
    }

    // ... rest of streaming code ...
  } catch (error) {
    if (error.name === 'TypeError' && error.message.includes('fetch')) {
      throw new Error('Network error. Please check your connection.');
    }
    throw error;
  }
}

Benefits of Streaming

• Better UX: Users see responses immediately
• Perceived Performance: Feels faster even if total time is similar
• Progressive Loading: Long responses don't feel like they're "stuck"
• Real-time Feel: More conversational and engaging

Next Steps

• Add React integration for component-based architecture
• Implement conversation history persistence
• Add typing indicators
• Customize streaming animation styles

See Also

• API Streaming Guide - Complete API reference
• Basic Chat Example - Non-streaming version
• React Integration - React-specific implementation