Streaming Chat Responses

Handle streaming responses from chatbots for real-time user experience.

Why Stream?

Benefit	Explanation
Faster UX	Users see text as it's generated
Better perception	Shortens perceived response time
Long content	Works better for generated stories, code
Cost feedback	Users can stop long generations early

Basic Streaming

Python Example

import requests

def chat_stream(chatbot_id, message):
    """Stream chatbot response"""
    response = requests.post(
        "http://localhost/api/v1/chatbot/chat",
        headers={"Authorization": "Bearer YOUR_API_KEY"},
        json={
            "chatbot_id": chatbot_id,
            "message": message,
            "stream": True
        },
        stream=True
    )

    print("Bot: ", end="", flush=True)

    for line in response.iter_lines():
        if line:
            data = line.decode('utf-8')
            if "response" in data:
                response_text = data.replace("response:", "").strip()
                print(response_text, end="", flush=True)

    print()  # Newline after complete

# Usage
chat_stream("your-chatbot-id", "Tell me a long story about AI")

JavaScript Example

async function chatStream(chatbotId, message) {
  const response = await fetch('http://localhost/api/v1/chatbot/chat', {
    method: 'POST',
    headers: {
      'Authorization': 'Bearer YOUR_API_KEY',
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({
      chatbot_id: chatbotId,
      message,
      stream: true
    })
  });

  if (!response.ok) {
    throw new Error('Stream request failed');
  }

  const reader = response.body.getReader();
  const decoder = new TextDecoder();

  process.stdout.write('Bot: ');

  while (true) {
    const { done, value } = await reader.read();
    if (done) break;

    const chunk = decoder.decode(value);
    const lines = chunk.split('\n');

    for (const line of lines) {
      if (line.startsWith('response:')) {
        const content = line.slice(9);
        process.stdout.write(content);
      }
    }
  }

  process.stdout.write('\n');
}

// Usage
chatStream('your-chatbot-id', 'Tell me about the future of AI')
  .catch(console.error);

Web Application Streaming

React Component

import React, { useState, useCallback, useRef } from 'react';

function StreamingChat() {
  const [chatbotId] = useState('your-chatbot-id');
  const [message, setMessage] = useState('');
  const [streamingResponse, setStreamingResponse] = useState('');
  const [isStreaming, setIsStreaming] = useState(false);
  const responseEndRef = useRef(null);

  const startStream = useCallback(async () => {
    if (!message.trim() || isStreaming) return;

    setIsStreaming(true);
    setStreamingResponse('');
    responseEndRef.current = null;

    try {
      const response = await fetch('http://localhost/api/v1/chatbot/chat', {
        method: 'POST',
        headers: {
          'Authorization': 'Bearer YOUR_API_KEY',
          'Content-Type': 'application/json'
        },
        body: JSON.stringify({
          chatbot_id: chatbotId,
          message,
          stream: true
        })
      });

      if (!response.ok) {
        throw new Error('Stream failed');
      }

      const reader = response.body.getReader();
      const decoder = new TextDecoder();

      while (true) {
        const { done, value } = await reader.read();
        if (done) break;

        const chunk = decoder.decode(value);
        const lines = chunk.split('\n');

        for (const line of lines) {
          if (line.startsWith('response:')) {
            const content = line.slice(9);
            setStreamingResponse(prev => prev + content);
          }
        }
      }

      setIsStreaming(false);
    } catch (error) {
      console.error('Stream error:', error);
      setIsStreaming(false);
    }
  }, [chatbotId, message, isStreaming]);

  const stopStream = () => {
    setIsStreaming(false);
    if (responseEndRef.current) {
      responseEndRef.current.abort();
    }
  };

  return (
    <div className="max-w-4xl mx-auto p-6">
      <div className="mb-4">
        <textarea
          value={message}
          onChange={(e) => setMessage(e.target.value)}
          className="w-full p-2 border rounded"
          rows="3"
          placeholder="Type your message..."
          disabled={isStreaming}
        />
      </div>

      <div className="space-x-2 mb-4">
        <button
          onClick={startStream}
          disabled={isStreaming || !message.trim()}
          className="px-6 py-2 bg-blue-500 text-white rounded disabled:opacity-50"
        >
          {isStreaming ? 'Streaming...' : 'Send'}
        </button>

        {isStreaming && (
          <button
            onClick={stopStream}
            className="px-4 py-2 bg-red-500 text-white rounded"
          >
            Stop
          </button>
        )}
      </div>

      <div className="border rounded-lg p-4 bg-gray-50 min-h-32">
        <pre className="whitespace-pre-wrap">
          {streamingResponse || <span className="text-gray-400">Response will appear here...</span>}
        </pre>
      </div>
    </div>
  );
}

export default StreamingChat;

Vue.js Component

<template>
  <div>
    <div class="mb-4">
      <textarea
        v-model="message"
        @keydown.enter="startStream"
        class="w-full p-2 border rounded"
        rows="3"
        placeholder="Type your message..."
        :disabled="isStreaming"
      />
    </div>

    <div class="space-x-2 mb-4">
      <button
        @click="startStream"
        :disabled="isStreaming || !message"
        class="px-6 py-2 bg-blue-500 text-white rounded disabled:opacity-50"
      >
        {{ isStreaming ? 'Streaming...' : 'Send' }}
      </button>

      <button
        v-if="isStreaming"
        @click="stopStream"
        class="px-4 py-2 bg-red-500 text-white rounded"
      >
        Stop
      </button>
    </div>

    <div class="border rounded-lg p-4 bg-gray-50 min-h-32">
      <pre class="whitespace-pre-wrap">
        {{ streamingResponse || 'Response will appear here...' }}
      </pre>
    </div>
  </div>
</template>

<script>
export default {
  data() {
    return {
      chatbotId: 'your-chatbot-id',
      message: '',
      streamingResponse: '',
      isStreaming: false,
      controller: null
    };
  },

  methods: {
    async startStream() {
      if (!this.message.trim() || this.isStreaming) return;

      this.isStreaming = true;
      this.streamingResponse = '';

      try {
        const response = await fetch('http://localhost/api/v1/chatbot/chat', {
          method: 'POST',
          headers: {
            'Authorization': 'Bearer YOUR_API_KEY',
            'Content-Type': 'application/json'
          },
          body: JSON.stringify({
            chatbot_id: this.chatbotId,
            message: this.message,
            stream: true
          })
        });

        if (!response.ok) {
          throw new Error('Stream failed');
        }

        this.controller = new AbortController();
        const reader = response.body.getReader();
        const decoder = new TextDecoder();

        while (true) {
          const { done, value } = await reader.read();
          if (done) break;

          const chunk = decoder.decode(value);
          const lines = chunk.split('\n');

          for (const line of lines) {
            if (line.startsWith('response:')) {
              const content = line.slice(9);
              this.streamingResponse += content;
            }
          }
        }

        this.isStreaming = false;
      } catch (error) {
        console.error('Stream error:', error);
        this.isStreaming = false;
      }
    },

    stopStream() {
      if (this.controller) {
        this.controller.abort();
      }
      this.isStreaming = false;
    }
  }
};
</script>

Streaming with Typing Effect

class TypingEffectStream {
  async chatWithTyping(chatbotId, message) {
    const response = await fetch('http://localhost/api/v1/chatbot/chat', {
      method: 'POST',
      headers: {
        'Authorization': 'Bearer YOUR_API_KEY',
        'Content-Type': 'application/json'
      },
      body: JSON.stringify({
        chatbot_id: chatbotId,
        message,
        stream: true
      })
    });

    const reader = response.body.getReader();
    const decoder = new TextDecoder();

    let wordBuffer = '';

    while (true) {
      const { done, value } = await reader.read();
      if (done) break;

      const chunk = decoder.decode(value);
      const lines = chunk.split('\n');

      for (const line of lines) {
        if (line.startsWith('response:')) {
          const newContent = line.slice(9);
          const words = newContent.split(' ');
          wordBuffer += newContent;

          // Type out words with natural delay
          while (wordBuffer.length > 0) {
            const words = wordBuffer.split(' ');
            const word = words.shift();
            process.stdout.write((wordBuffer.length > 0 ? ' ' : '') + word);
            wordBuffer = words.join(' ');

            if (wordBuffer.length < 5) {
              await new Promise(r => setTimeout(r, 50));  // Fast typing
            } else if (wordBuffer.length < 15) {
              await new Promise(r => setTimeout(r, 80));  // Medium typing
            } else {
              await new Promise(r => setTimeout(r, 120));  // Slow typing for punctuation
            }
          }
        }
      }
    }
}

// Usage
const streamer = new TypingEffectStream();
streamer.chatWithTyping('your-chatbot-id', 'Tell me a long story')
  .catch(console.error);

Stream Completion Detection

async function chatWithCompletionDetection(chatbotId, message) {
  let fullResponse = '';
  let complete = false;

  const response = await fetch('http://localhost/api/v1/chatbot/chat', {
    method: 'POST',
    headers: {
      'Authorization': 'Bearer YOUR_API_KEY',
      'Content-Type': 'application/json'
    },
    body: JSON.stringify({
      chatbot_id: chatbotId,
      message,
      stream: true
    })
  });

  const reader = response.body.getReader();
  const decoder = new TextDecoder();

  while (!complete) {
    const { done, value } = await reader.read();
    if (done) break;

    const chunk = decoder.decode(value);
    const lines = chunk.split('\n');

    for (const line of lines) {
      if (line.startsWith('data: ')) {
        const data = JSON.parse(line.slice(5));

        if (data.finish_reason) {
          complete = true;
        }

        if (data.choices && data.choices[0]?.delta?.content) {
          const content = data.choices[0].delta.content;
          fullResponse += content;
          process.stdout.write(content);
        }
      }
    }
  }

  process.stdout.write('\n');
  console.log('Complete response:', fullResponse);
}

chatWithCompletionDetection('your-chatbot-id', 'Hello!')
  .catch(console.error);

Best Practices

Always Use Conversation IDs - Maintain context across streaming requests
Handle Errors - Implement proper error handling for streams
Stop Capability - Allow users to cancel long generations
Buffer Display - Show text as it arrives for better UX
Complete Detection - Detect when stream is finished
Timeout Handling - Set appropriate timeouts to prevent hanging

Next Steps

Python Integration - Python streaming examples
JavaScript Integration - More JavaScript examples
Chatting - Complete chat API documentation
API Reference - Full endpoint documentation

Why Stream?​

Basic Streaming​

Python Example​

JavaScript Example​

Web Application Streaming​

React Component​

Vue.js Component​

Streaming with Typing Effect​

Stream Completion Detection​

Best Practices​

Next Steps​

Why Stream?

Basic Streaming

Python Example

JavaScript Example

Web Application Streaming

React Component

Vue.js Component

Streaming with Typing Effect

Stream Completion Detection

Best Practices

Next Steps