AI Streaming Chat
Full Screen DemoA real-time streaming chat interface that shows AI responses as they’re generated, token by token. This creates a more engaging user experience compared to waiting for the entire response.
defmodule PetalProWeb.StreamingChatLive do
@moduledoc """
LiveView for a streaming AI chat interface.
This implementation demonstrates best practices for handling real-time
AI responses using a unified message structure that transitions smoothly
from streaming to completed states.
"""
use PetalProWeb, :live_view
import PetalProWeb.Components.StreamingChatComponents
@impl true
def mount(_params, _session, socket) do
socket =
assign(socket,
page_title: "Streaming AI Chat",
form: to_form(%{}, as: :form),
messages: []
)
{:ok, socket}
end
@impl true
def handle_event("submit", %{"form" => %{"message" => message}}, socket) do
case PetalPro.AI.StreamingChat.validate_message(message) do
{:ok, clean_message} ->
# Add user message
user_message = %{role: "user", content: clean_message}
# Add empty streaming assistant message
assistant_message = %{role: "assistant", content: "", streaming: true}
messages = socket.assigns.messages ++ [user_message, assistant_message]
# Get conversation history (only completed messages)
conversation = get_conversation_history(socket.assigns.messages ++ [user_message])
# Start streaming task
parent_pid = self()
Task.start(fn ->
PetalPro.AI.StreamingChat.stream_response(conversation, parent_pid,
model: "gpt-4o-mini",
temperature: 0.7
)
end)
socket =
socket
|> assign(messages: messages, form: to_form(%{}, as: :form))
{:noreply, socket}
{:error, reason} ->
{:noreply, put_flash(socket, :error, reason)}
end
end
@impl true
def handle_event("clear", _params, socket) do
{:noreply, assign(socket, messages: [])}
end
@impl true
def handle_info({:stream_chunk, chunk}, socket) do
# Update the last message (which is streaming)
messages =
List.update_at(socket.assigns.messages, -1, fn msg ->
%{msg | content: msg.content <> chunk}
end)
{:noreply, assign(socket, messages: messages)}
end
@impl true
def handle_info(:stream_done, socket) do
# Mark the last message as no longer streaming
messages =
List.update_at(socket.assigns.messages, -1, fn msg ->
if msg.content != "" do
Map.delete(msg, :streaming)
else
msg
end
end)
# If the last message is still empty, remove it and show error
last_message = List.last(messages)
if last_message.content == "" do
socket =
socket
|> assign(messages: Enum.drop(messages, -1))
|> put_flash(:error, "No response received from AI. Please try again.")
{:noreply, socket}
else
{:noreply, assign(socket, messages: messages)}
end
end
@impl true
def handle_info({:stream_error, error}, socket) do
message = error_to_message(error)
# Remove the streaming message
messages = Enum.drop(socket.assigns.messages, -1)
socket =
socket
|> assign(messages: messages)
|> put_flash(:error, message)
{:noreply, socket}
end
# Private functions
defp streaming?(messages) do
Enum.any?(messages, &Map.get(&1, :streaming, false))
end
defp get_conversation_history(messages) do
# Extract just role and content for API calls, excluding streaming messages
messages
|> Enum.reject(&Map.get(&1, :streaming, false))
|> Enum.map(&Map.take(&1, [:role, :content]))
end
defp error_to_message({:api_error, error}) do
error_str = inspect(error)
cond do
String.contains?(error_str, "closed") ->
"Connection closed by server. You may have hit OpenAI's rate limit. Please wait a few seconds before sending another message."
String.contains?(error_str, ["timeout", "Timeout"]) ->
"Request timed out. Please try again."
true ->
"API Error: #{error_str}"
end
end
defp error_to_message({:timeout, _reason}) do
"Request timed out. Please try again."
end
defp error_to_message({:exit, _reason}) do
"Connection lost. Please try again."
end
defp error_to_message({:unexpected, reason}) do
"An error occurred: #{reason}"
end
defp error_to_message(error) do
"Error: #{inspect(error)}"
end
end
<.container class="max-w-4xl my-10">
<.card class="mb-5 p-4">
<.messages_container messages={@messages} />
</.card>
<div class="flex justify-end mb-4">
<.button
:if={@messages != []}
size="sm"
color="white"
phx-click="clear"
label="Clear Chat"
icon="hero-trash"
/>
</div>
<.form for={@form} phx-submit="submit" id="chat-form" class="flex gap-2">
<div class="flex-1">
<.input
field={@form[:message]}
type="text"
placeholder="Type your message..."
id="chat-input"
aria-label="Chat message input"
phx-hook="AutoFocusHook"
/>
</div>
<.button
type="submit"
color="primary"
size="sm"
disabled={streaming?(@messages)}
phx-disable-with="Sending..."
icon="hero-paper-airplane"
label={if streaming?(@messages), do: "Sending...", else: "Send"}
aria-label="Send message"
/>
</.form>
<div class="mt-4 text-sm text-gray-500 dark:text-gray-400">
<.icon name="hero-information-circle" class="inline w-4 h-4" />
Messages stream in real-time as the AI generates them
</div>
</.container>
/**
* LiveView Hooks for Streaming Chat
*
* This file contains all client-side hooks for the streaming chat feature.
* Import these hooks in your app.js file and register them with LiveSocket.
*/
/**
* AutoScrollHook
*
* Attaches to the messages container and automatically scrolls to the bottom
* when new messages are added. This hook is more efficient than listening to
* global phx:update events since it only runs when the specific container updates.
*
* This is a "smart scroll" implementation - it only auto-scrolls if the user
* is already near the bottom, allowing them to scroll up to read earlier
* messages without being interrupted.
*
* Usage:
* <div id="chat-messages" phx-hook="AutoScrollHook">
* <!-- messages here -->
* </div>
*
* How it works:
* - mounted(): Scrolls to bottom on initial load
* - updated(): Scrolls to bottom when new messages arrive (if near bottom)
*/
export const AutoScrollHook = {
mounted() {
this.scrollToBottom();
},
updated() {
this.scrollToBottom();
},
scrollToBottom() {
const scrollTop = this.el.scrollTop;
const scrollHeight = this.el.scrollHeight;
const clientHeight = this.el.clientHeight;
const distanceFromBottom = scrollHeight - scrollTop - clientHeight;
// Only auto-scroll if user is near the bottom (within 150px)
const isNearBottom = distanceFromBottom < 150;
if (isNearBottom) {
requestAnimationFrame(() => {
this.el.scrollTop = this.el.scrollHeight;
});
}
}
};
/**
* AutoFocusHook
*
* Enhances the chat input field with two key behaviors:
*
* 1. Clears the input when the form is submitted, removing the need for
* the server to manage form state clearing
* 2. Maintains focus on the input after submission, providing a smooth UX
* so users can immediately type their next message
*
* This hook demonstrates best practices for client-side form handling:
* - Clear input on submit event (immediate feedback)
* - Maintain focus for continuous interaction
* - Also handle the updated lifecycle in case of edge cases
*
* Usage:
* <input type="text" phx-hook="AutoFocusHook" />
*
* How it works:
* - mounted(): Focuses input on initial render and attaches form submit handler
* - updated(): Clears and refocuses input if the element is re-rendered
* by LiveView (e.g., due to validation errors)
*/
export const AutoFocusHook = {
mounted() {
this.el.focus();
const form = this.el.closest("form");
if (form) {
form.addEventListener("submit", () => {
setTimeout(() => {
this.el.value = "";
this.el.focus();
}, 0);
});
}
},
updated() {
this.el.value = "";
this.el.focus();
}
};
defmodule PetalProWeb.Components.StreamingChatComponents do
@moduledoc """
Components for rendering streaming chat messages.
These components handle the display of chat messages including real-time
streaming updates from AI assistants.
"""
use PetalProWeb, :component
@doc """
Renders a single chat message.
## Attributes
- `:message` - A map with `:role`, `:content`, and optionally `:streaming` fields
- `:role` can be "user" or "assistant"
- `:streaming` indicates if the message is currently being streamed
"""
attr :message, :map, required: true
def chat_message(assigns) do
~H"""
<div class="flex gap-3">
<%!-- Spacer for user messages (right-aligned) --%>
<div :if={@message.role == "user"} class="flex-1"></div>
<div class={[
"flex gap-3 max-w-[85%]",
@message.role == "user" && "flex-row-reverse"
]}>
<%!-- Avatar --%>
<div class={[
"flex-shrink-0 w-8 h-8 rounded-full flex items-center justify-center",
@message.role == "user" && "bg-primary-500",
@message.role == "assistant" && "bg-gray-700"
]}>
<.icon
name={if @message.role == "user", do: "hero-user", else: "hero-cpu-chip"}
class="w-5 h-5 text-white"
/>
</div>
<%!-- Message bubble --%>
<div class={[
"px-4 py-3 rounded-lg",
@message.role == "user" && "bg-primary-500 text-white",
@message.role == "assistant" &&
"bg-gray-200 dark:bg-gray-700 text-gray-900 dark:text-gray-100"
]}>
<%= if Map.get(@message, :streaming) && (@message.content == "" || @message.content == nil) do %>
<%!-- Show spinner while waiting for first chunk --%>
<.icon name="hero-cog-6-tooth-solid" class="w-4 h-4 text-gray-400 animate-spin" />
<% else %>
<div class="text-sm whitespace-pre-wrap">{@message.content}</div>
<% end %>
</div>
</div>
<%!-- Spacer for assistant messages (left-aligned) --%>
<div :if={@message.role == "assistant"} class="flex-1"></div>
</div>
"""
end
@doc """
Renders a scrollable container for chat messages.
## Attributes
- `:messages` - List of message maps to display
- `:empty_state` - Optional slot for custom empty state content
"""
attr :messages, :list, required: true
slot :empty_state
def messages_container(assigns) do
~H"""
<div
class="space-y-4 min-h-[300px] max-h-[350px] overflow-y-auto p-4"
id="chat-messages"
phx-hook="AutoScrollHook"
>
<%= if @messages == [] do %>
<%= if @empty_state != [] do %>
<%= render_slot(@empty_state) %>
<% else %>
<div class="flex items-center justify-center h-[300px]">
<div class="text-center text-gray-500 dark:text-gray-400">
<.icon name="hero-chat-bubble-left-right" class="w-16 h-16 mx-auto mb-4 opacity-50" />
<p class="text-lg font-medium">Start a conversation</p>
<p class="text-sm">Ask me anything!</p>
</div>
</div>
<% end %>
<% else %>
<div :for={{message, index} <- Enum.with_index(@messages)} id={"message-#{index}"}>
<.chat_message message={message} />
</div>
<% end %>
</div>
"""
end
end
defmodule PetalPro.AI.StreamingChat do
@moduledoc """
Handles streaming chat responses with proper message validation.
"""
def stream_response(messages, parent, opts \\ []) do
system_message = %{
role: "system",
content: "You are a helpful AI assistant."
}
conversation = [system_message | messages]
model = Keyword.get(opts, :model, "gpt-4o-mini")
temperature = Keyword.get(opts, :temperature, 0.7)
case PetalPro.AI.OpenAI.chat_stream(conversation,
model: model, temperature: temperature) do
{:ok, stream} ->
stream
|> Enum.each(fn chunk ->
send(parent, {:stream_chunk, chunk})
end)
send(parent, :stream_done)
{:error, error} ->
send(parent, {:stream_error, error})
end
end
def validate_message(content) when is_binary(content) do
trimmed = String.trim(content)
if byte_size(trimmed) == 0 do
{:error, "Message cannot be empty"}
else
{:ok, trimmed}
end
end
def validate_message(_), do: {:error, "Invalid message format"}
end
// In your app.js file, import and register the hooks
import { AutoScrollHook, AutoFocusHook } from "./hooks/streaming-chat-hooks"
let Hooks = {}
Hooks.AutoScrollHook = AutoScrollHook
Hooks.AutoFocusHook = AutoFocusHook
let liveSocket = new LiveSocket("/live", Socket, {
hooks: Hooks,
params: {_csrf_token: csrfToken}
})
// Connect the LiveSocket
liveSocket.connect()
defmodule PetalPro.AI.OpenAI do
def chat_stream(messages, opts \\ []) do
api_key = System.get_env("OPENAI_API_KEY")
unless api_key do
{:error, "OPENAI_API_KEY not set"}
else
ReqLLM.put_key(:openai_api_key, api_key)
model = Keyword.get(opts, :model, "gpt-4o-mini")
temperature = Keyword.get(opts, :temperature, 0.7)
model_spec = "openai:#{model}"
case ReqLLM.stream_text(model_spec, messages,
temperature: temperature) do
{:ok, stream_response} ->
token_stream = ReqLLM.StreamResponse.tokens(stream_response)
{:ok, token_stream}
{:error, error} ->
{:error, "Request failed: #{inspect(error)}"}
end
end
end
end
live(
"/recipes/streaming-chat/examples/streaming-chat",
StreamingChatLive,
:index
)
def deps do
[
{:req, "~> 0.5"},
{:req_llm, "~> 1.0"}
]
end
Key Features
- Real-time token streaming using ReqLLM's streaming API
- Message validation and sanitization
- Reusable message components for clean templates
- Multi-turn conversations with full context history
- Comprehensive error handling with retry logic
- Auto-scroll to latest message with visual indicators
How It Works
- User submits a message, which is validated for safety
- Message is added to the conversation history
- A background Task is started to stream the AI response
-
Each token chunk is sent to the LiveView via
handle_infomessages - The streaming message updates in real-time using reusable components
- Once complete, the assistant message is added to the conversation history
- Full conversation context is maintained for multi-turn interactions