http://127.0.0.1:1234/v1
Change make sure you are using correct
model wih full path
I'm using
qwen/qwen3.5-9b
import requests
from rich.console import Console
from dataclasses import dataclass, field
from typing import Any
@dataclass
class Agent:
model: str = 'qwen/qwen3.5-9b'
base_url: str = "http://127.0.0.1:1234/v1"
api_key: str = field(default="NO_API_KEY",repr=False)
messages: list[dict[str,Any]] = field(default_factory=list)
def __post_init__(self) -> None:
self.base_url = self.base_url.rstrip('/')
def chat(self, user_message: str) -> str:
self.messages.append({"role":"user","content": user_message})
url = f"{self.base_url}/chat/completions/"
headers = {
"Authorization": f"Bearer {self.api_key}",
"Content-Type": "application/json",
}
r = requests.post(
url,
headers=headers,
json={"model": self.model, "messages": self.messages},
timeout=300,
)
r.raise_for_status()
data = r.json()
choices = data.get("choices")
if not choices:
raise RuntimeError("Model response missing choices ")
message = choices[0].get("message")
if message is None:
raise RuntimeError("Model response no message")
response = message.get("content") or ""
self.messages.append({"role": "assistant","content": response})
return response
def main() -> None:
agent = Agent(model = 'qwen/qwen3.5-9b')
console = Console()
while True:
console.print("[green]User:[/green]", end="")
user_input = console.input()
if user_input.strip().lower() in {":q","quit","exit"}:
console.print("[dim]peace out my man.[/dim]")
return
with console.status("[dim]Thinking...[/dim]",spinner="arc"):
response = agent.chat(user_input).strip()
console.print(f"[blue]Assistant:[/blue]",{response})
if __name__ == '__main__':
main()