Best LLM for AI Agents: Test and Switch Models in Python

# The only line that knows which model runs the agent.
MODEL = os.environ.get("AGENT_MODEL", "openai:gpt-5-mini")


async def main():
    agent = await build_agent(
        model=MODEL,
        servers={"shop": StdioServerSpec(command=sys.executable, args=["shop_server.py"])},

AGENT_MODEL=openai:gpt-4.1-mini python agent.py
AGENT_MODEL=anthropic:claude-sonnet-4-5 python agent.py

pip install promptise
export OPENAI_API_KEY="sk-..."

from promptise.mcp.server import MCPServer

server = MCPServer("shop")

# A small, fixed dataset, so every eval task has one right answer.
PRODUCTS = {
    "MUG-01": {"name": "Ceramic mug", "price_eur": 12.50, "stock": 42},
    "CUP-02": {"name": "Travel cup", "price_eur": 9.90, "stock": 7},
    "LAMP-03": {"name": "Desk lamp", "price_eur": 39.00, "stock": 15},
    "LAMP-04": {"name": "Floor lamp", "price_eur": 89.00, "stock": 4},
    "DESK-05": {"name": "Standing desk", "price_eur": 499.00, "stock": 3},
}
CUSTOMERS = {
    "C-7": {"name": "Dana Ito", "country": "PT", "tier": "gold"},
    "C-8": {"name": "Lee Park", "country": "DE", "tier": "standard"},
    "C-9": {"name": "Sam Ruiz", "country": "US", "tier": "standard"},
}
ORDERS = {
    "A-1001": {"customer_id": "C-8", "items": [{"sku": "MUG-01", "qty": 2}], "status": "shipped"},
    "A-1002": {"customer_id": "C-7", "items": [{"sku": "LAMP-03", "qty": 1}], "status": "processing"},
    "A-1003": {"customer_id": "C-7", "items": [{"sku": "DESK-05", "qty": 1}], "status": "delivered"},
    "A-1004": {"customer_id": "C-7", "items": [{"sku": "CUP-02", "qty": 3}], "status": "cancelled"},
    "A-1005": {"customer_id": "C-9", "items": [{"sku": "LAMP-04", "qty": 1}, {"sku": "MUG-01", "qty": 1}], "status": "shipped"},
}
EUR_RATES = {"USD": 1.087, "GBP": 0.842, "CHF": 0.937}
SHIPPING = {"DE": (4.90, 1.20), "PT": (6.90, 2.10), "US": (19.00, 4.50)}  # base fee, per kg


@server.tool()
async def get_order(order_id: str) -> dict:
    """Get an order: customer ID, items (SKU and quantity), status and total in EUR."""
    order = ORDERS.get(order_id.strip().upper())
    if order is None:
        return {"error": f"No order {order_id}."}
    total = sum(PRODUCTS[i["sku"]]["price_eur"] * i["qty"] for i in order["items"])
    return {"order_id": order_id.strip().upper(), **order, "total_eur": round(total, 2)}


@server.tool()
async def list_orders(customer_id: str) -> list[str]:
    """List the IDs of every order a customer has placed, including cancelled ones."""
    return [oid for oid, o in ORDERS.items() if o["customer_id"] == customer_id.strip().upper()]


@server.tool()
async def get_customer(customer_id: str) -> dict:
    """Get a customer: name, two-letter country code and loyalty tier."""
    customer = CUSTOMERS.get(customer_id.strip().upper())
    if customer is None:
        return {"error": f"No customer {customer_id}."}
    return {"customer_id": customer_id.strip().upper(), **customer}


@server.tool()
async def get_product(sku: str) -> dict:
    """Get a product by SKU: name, unit price in EUR and units in stock."""
    product = PRODUCTS.get(sku.strip().upper())
    if product is None:
        return {"error": f"No product {sku}."}
    return {"sku": sku.strip().upper(), **product}


@server.tool()
async def search_products(query: str) -> list[dict]:
    """Find products whose name contains the query. Returns SKU and name only."""
    q = query.lower().rstrip("s")
    return [{"sku": s, "name": p["name"]} for s, p in PRODUCTS.items() if q in p["name"].lower()]


@server.tool()
async def convert_from_eur(amount_eur: float, currency: str) -> dict:
    """Convert an amount in EUR to USD, GBP or CHF at today's rate."""
    rate = EUR_RATES.get(currency.strip().upper())
    if rate is None:
        return {"error": f"No rate for {currency}. Supported: USD, GBP, CHF."}
    return {"amount_eur": amount_eur, "currency": currency.upper(), "rate": rate, "amount": round(amount_eur * rate, 2)}


@server.tool()
async def shipping_cost(country: str, weight_kg: float) -> dict:
    """Shipping cost in EUR for a parcel to a country (two-letter code, for example "DE")."""
    fees = SHIPPING.get(country.strip().upper())
    if fees is None:
        return {"error": f"We don't ship to {country}."}
    base, per_kg = fees
    return {"country": country.upper(), "weight_kg": weight_kg, "cost_eur": round(base + per_kg * weight_kg, 2)}


if __name__ == "__main__":
    server.run()

import asyncio
import os
import sys

from promptise import build_agent
from promptise.config import StdioServerSpec

# The only line that knows which model runs the agent.
MODEL = os.environ.get("AGENT_MODEL", "openai:gpt-5-mini")


async def main():
    agent = await build_agent(
        model=MODEL,
        servers={"shop": StdioServerSpec(command=sys.executable, args=["shop_server.py"])},
        instructions="You are a shop assistant. Look everything up with your tools; never guess.",
        trace_tools=True,
    )
    try:
        result = await agent.ainvoke(
            {"messages": [{"role": "user", "content": "Where is order A-1001, and who ordered it?"}]}
        )
        print(f">>> [{MODEL}]", result["messages"][-1].content)
    finally:
        await agent.shutdown()


asyncio.run(main())

python agent.py
AGENT_MODEL=openai:gpt-4.1-mini python agent.py

import asyncio
import json
import re
import statistics
import sys
import time

from promptise import build_agent
from promptise.config import StdioServerSpec

MODELS = sys.argv[1:] or ["openai:gpt-5-mini"]
REPEATS = 3

INSTRUCTIONS = (
    "You are a shop assistant. Look everything up with your tools; never guess. "
    "End your reply with one line: 'Answer: <value>', where the value is only a number "
    "(no currency sign), a SKU, a two-letter country code or a single word."
)

# Ten tasks with one right answer each: single lookups, a "we can't", chains of
# 2-4 calls, parallel calls, filtering and a little arithmetic.
TASKS = [
    ("What's the status of order A-1005?", "shipped"),
    ("Can we ship a 1 kg parcel to Japan? Answer yes or no.", "no"),
    ("Which country does the customer behind order A-1002 live in?", "PT"),
    ("What is the total of order A-1003 in US dollars?", 542.41),
    ("How many orders has customer C-7 placed, including cancelled ones?", 3),
    ("Which is cheaper per unit, MUG-01 or CUP-02? Answer with the SKU.", "CUP-02"),
    ("How much does it cost to ship a 2.5 kg parcel to the customer of order A-1001?", 7.90),
    ("What is the combined stock of every lamp we sell?", 19),
    ("What is the total value in EUR of customer C-7's orders that were not cancelled?", 538.00),
    ("Customer C-9 wants one floor lamp shipped to them; the parcel weighs 4 kg. "
     "What do they pay in Swiss francs for the lamp plus shipping?", 118.06),
]


def grade(reply: str, expected) -> tuple[bool, str]:
    """Pull the 'Answer:' line out of the reply and compare it with the known answer."""
    found = re.findall(r"answer:\s*(.+)", reply, flags=re.IGNORECASE)
    if not found:
        return False, "(no Answer line)"
    value = found[-1].strip().strip("*`. ")
    if isinstance(expected, str):
        return value.upper() == expected.upper(), value
    number = re.search(r"-?\d[\d,]*\.?\d*", value)
    if number is None:
        return False, value
    return abs(float(number.group().replace(",", "")) - expected) < 0.006, value


async def run_task(agent, prompt: str) -> dict:
    start = time.perf_counter()
    result = await agent.ainvoke({"messages": [{"role": "user", "content": prompt}]})
    seconds = time.perf_counter() - start
    messages = result["messages"]
    usage = [m.usage_metadata for m in messages if getattr(m, "usage_metadata", None)]
    tools = [m for m in messages if getattr(m, "type", "") == "tool"]
    return {
        "seconds": seconds,
        "input_tokens": sum(u["input_tokens"] for u in usage),
        "output_tokens": sum(u["output_tokens"] for u in usage),
        "tool_calls": len(tools),
        "reply": messages[-1].content,
    }


async def evaluate(model: str, log) -> dict:
    agent = await build_agent(
        model=model,
        servers={"shop": StdioServerSpec(command=sys.executable, args=["shop_server.py"])},
        instructions=INSTRUCTIONS,
    )
    runs = []
    try:
        for repeat in range(REPEATS):
            for number, (prompt, expected) in enumerate(TASKS, 1):
                run = await run_task(agent, prompt)
                run["correct"], run["answer"] = grade(run["reply"], expected)
                run.update(model=model, task=number, repeat=repeat, expected=expected)
                log.write(json.dumps(run) + "\n")
                runs.append(run)
                mark = "ok  " if run["correct"] else "MISS"
                print(f"{model} task {number:2} run {repeat + 1}: {mark} {run['answer']!r} (expected {expected!r})", flush=True)
    finally:
        await agent.shutdown()
    return {
        "model": model,
        "correct": sum(r["correct"] for r in runs),
        "runs": len(runs),
        "median_s": statistics.median(r["seconds"] for r in runs),
        "input_tokens": statistics.mean(r["input_tokens"] for r in runs),
        "output_tokens": statistics.mean(r["output_tokens"] for r in runs),
        "tool_calls": statistics.mean(r["tool_calls"] for r in runs),
    }


async def main():
    with open("eval_runs.jsonl", "a") as log:
        summaries = [await evaluate(model, log) for model in MODELS]
    print(f"\n{'model':<22}{'correct':>10}{'median s':>10}{'in tok':>9}{'out tok':>9}{'tools':>7}")
    for s in summaries:
        print(
            f"{s['model']:<22}{s['correct']:>4}/{s['runs']:<5}{s['median_s']:>10.1f}"
            f"{s['input_tokens']:>9.0f}{s['output_tokens']:>9.0f}{s['tool_calls']:>7.1f}"
        )


asyncio.run(main())

python eval_models.py openai:gpt-5-mini openai:gpt-5 openai:gpt-5-nano openai:gpt-4.1-mini openai:gpt-4.1-nano

promptise models check openai:gpt-5-mini --ping
promptise models check anthropic:claude-sonnet-4-5

promptise models check azure:chat-prod

promptise models env bedrock

import os
import sys

from promptise import check_model

# Check the configuration before deploying. No model is called.
spec = os.environ.get("AGENT_MODEL", "openai:gpt-5-mini")
result = check_model(spec)
print(f"{spec} -> {result.canonical}, ok={result.ok}")
for problem in result.problems:
    print("  -", problem)
sys.exit(0 if result.ok else 1)

"""Configure six providers in code and show where each one would send requests.

Nothing here calls a model: resolve() only builds the client. Only an OpenAI key
exists on this machine, so the other credentials are placeholders.
"""
import os

from promptise import Model

MODELS = [
    Model("gpt-5-mini", provider="openai", api_key=os.environ["OPENAI_API_KEY"]),
    Model("claude-sonnet-4-5", provider="anthropic", api_key="sk-ant-placeholder"),
    Model(
        "gpt-4o",
        provider="azure",
        deployment="chat-prod",
        endpoint="https://my-resource.openai.azure.com/",
        api_key="placeholder",
        api_version="2024-10-21",
    ),
    Model("anthropic.claude-sonnet-4-20250514-v1:0", provider="bedrock", region="eu-central-1", api_key="ABSK-placeholder"),
    Model("gemini-2.5-pro", provider="gemini", api_key="AIza-placeholder"),
    Model("llama3.1", provider="ollama"),
]

for model in MODELS:
    llm = model.resolve()
    url = (
        getattr(llm, "azure_endpoint", None)
        or getattr(llm, "openai_api_base", None)
        or getattr(llm, "anthropic_api_url", None)
        or "(the OpenAI default)"
    )
    print(f"{model.spec:<55} {type(llm).__name__:<16} {url}")

print(repr(MODELS[2]))

"""A stub that speaks just enough of the OpenAI chat API to test the wiring.

It is not a model. It asks for one tool call, then echoes the tool's result.
"""
import json
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer


class ChatCompletions(BaseHTTPRequestHandler):
    def do_POST(self):
        body = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
        tools = [t["function"]["name"] for t in body.get("tools", [])]
        last = body["messages"][-1]
        print(f"POST {self.path} model={body['model']} tools={tools} last={last['role']}", flush=True)

        if self.path != "/v1/chat/completions":
            self.send_error(404)
            return
        if last["role"] == "tool":
            message = {"role": "assistant", "content": f"Stub reply. The tool returned: {last['content']}"}
            finish = "stop"
        else:
            call = {"id": "call_1", "type": "function",
                    "function": {"name": "get_order", "arguments": json.dumps({"order_id": "A-1001"})}}
            message = {"role": "assistant", "content": None, "tool_calls": [call]}
            finish = "tool_calls"

        reply = json.dumps({
            "id": "chatcmpl-stub", "object": "chat.completion", "created": int(time.time()),
            "model": body["model"],
            "choices": [{"index": 0, "message": message, "finish_reason": finish}],
            "usage": {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0},
        }).encode()
        self.send_response(200)
        self.send_header("Content-Type", "application/json")
        self.send_header("Content-Length", str(len(reply)))
        self.end_headers()
        self.wfile.write(reply)

    def log_message(self, *args):
        pass  # one line per request is printed above


if __name__ == "__main__":
    ThreadingHTTPServer(("127.0.0.1", 8400), ChatCompletions).serve_forever()

OPENAI_BASE_URL=http://127.0.0.1:8400/v1 OPENAI_API_KEY=local AGENT_MODEL=openai:stub-model python agent.py

promptise models check ollama:llama3.1

import asyncio
import sys

from promptise import Model, build_agent
from promptise.config import StdioServerSpec

MODELS = {
    # Any server that speaks the OpenAI chat API: vLLM, LM Studio, a gateway, or our stub.
    "OpenAI-compatible": Model("stub-model", provider="openai", endpoint="http://127.0.0.1:8400/v1", api_key="local"),
    # The Ollama route, pointed at the same stub instead of a real Ollama.
    "Ollama route": Model("llama3.1", provider="ollama", endpoint="http://127.0.0.1:8400"),
}


async def main():
    for label, model in MODELS.items():
        agent = await build_agent(
            model=model,
            servers={"shop": StdioServerSpec(command=sys.executable, args=["shop_server.py"])},
            trace_tools=True,
        )
        try:
            result = await agent.ainvoke({"messages": [{"role": "user", "content": "Where is order A-1001?"}]})
            print(f">>> {label}: {result['messages'][-1].content}\n")
        finally:
            await agent.shutdown()


asyncio.run(main())