homepage

1

http://127.0.0.1:1234/v1

Change make sure you are using correct model wih full path

I'm using

qwen/qwen3.5-9b

** make this script full height
import requests 
from rich.console import Console
from dataclasses import dataclass, field
from typing import Any

@dataclass
class Agent:
    model: str = 'qwen/qwen3.5-9b'
    base_url: str = "http://127.0.0.1:1234/v1"
    api_key: str = field(default="NO_API_KEY",repr=False)
    messages: list[dict[str,Any]] = field(default_factory=list)

    def __post_init__(self) -> None:
        self.base_url = self.base_url.rstrip('/')

    def chat(self, user_message: str) -> str:
        self.messages.append({"role":"user","content": user_message})
        url = f"{self.base_url}/chat/completions/"

        headers = {
            "Authorization": f"Bearer {self.api_key}",
            "Content-Type": "application/json",
        }
        r = requests.post(
            url,
            headers=headers,
            json={"model": self.model, "messages": self.messages},
            timeout=300,
        )
        r.raise_for_status()
        data = r.json()
        choices = data.get("choices")

        if not choices:
            raise RuntimeError("Model response missing choices ")

        message = choices[0].get("message") 
        if message is None:
            raise RuntimeError("Model response no message")
        response = message.get("content") or ""
        self.messages.append({"role": "assistant","content": response})
        return response
def main() -> None:
    agent = Agent(model = 'qwen/qwen3.5-9b')
    console = Console()

    while True:
        console.print("[green]User:[/green]", end="")
        user_input = console.input()
        if user_input.strip().lower() in {":q","quit","exit"}:
            console.print("[dim]peace out my man.[/dim]")
            return
        with console.status("[dim]Thinking...[/dim]",spinner="arc"):
            response = agent.chat(user_input).strip()
        console.print(f"[blue]Assistant:[/blue]",{response})
    
    

if __name__ == '__main__':
    main()