d17462edc4
- llm.py: _create() wrapper retries APIConnectionError (incl. timeout) up to LLM_MAX_ATTEMPTS=3 with LLM_RETRY_DELAY_SECONDS=5s backoff; 4xx still fails fast. The local LLM server is shared with Hermes and serializes requests, so a request can be dropped by the client timeout while queued — chat/send was 500ing on that. - helpers.py: internal_error() now logs full exception detail server-side (was only the type name), so 500s are diagnosable; client message unchanged. - config.py: LLM_MAX_ATTEMPTS / LLM_RETRY_DELAY_SECONDS env knobs. - tests: test_llm_retry_queue (transient retried, 4xx not retried, no retry on success), test_internal_error_logging (detail in log, hidden from client, order-independent via explicit handler + logger re-enable).
96 lines
2.5 KiB
Python
96 lines
2.5 KiB
Python
"""LLM client: transient failures (timeout / connection reset — what a shared,
|
|
queueing LLM server produces under load) are retried so chat survives the queue;
|
|
a 4xx from the server (bad key / model) is NOT retried and fails fast.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from app.llm import LLMClient, LLMError
|
|
from openai import APIStatusError, APITimeoutError
|
|
|
|
|
|
def _make_client(max_attempts: int = 3) -> LLMClient:
|
|
c = LLMClient(base_url="http://localhost:9/v1", api_key="test-key", model="m")
|
|
c.max_attempts = max_attempts
|
|
c.retry_delay = 0.0
|
|
return c
|
|
|
|
|
|
def _resp():
|
|
message = type("M", (), {"content": "ok"})()
|
|
choice = type("C", (), {"message": message})()
|
|
return type("R", (), {"choices": [choice]})()
|
|
|
|
|
|
def _patch_sleep(monkeypatch):
|
|
import app.llm as llm
|
|
|
|
monkeypatch.setattr(llm.time, "sleep", lambda s: None)
|
|
|
|
|
|
def test_transient_failure_retries_then_succeeds(monkeypatch):
|
|
c = _make_client(max_attempts=3)
|
|
_patch_sleep(monkeypatch)
|
|
calls = {"n": 0}
|
|
|
|
def flaky(**kwargs):
|
|
calls["n"] += 1
|
|
if calls["n"] < 3:
|
|
raise APITimeoutError(request="req")
|
|
return _resp()
|
|
|
|
c.client.chat.completions.create = flaky
|
|
assert c.complete("sys", "user") == "ok"
|
|
assert calls["n"] == 3
|
|
|
|
|
|
def test_persistent_transient_failure_raises_after_max_attempts(monkeypatch):
|
|
c = _make_client(max_attempts=3)
|
|
_patch_sleep(monkeypatch)
|
|
calls = {"n": 0}
|
|
|
|
def always_timeout(**kwargs):
|
|
calls["n"] += 1
|
|
raise APITimeoutError(request="req")
|
|
|
|
c.client.chat.completions.create = always_timeout
|
|
with pytest.raises(LLMError):
|
|
c.complete("sys", "user")
|
|
assert calls["n"] == 3
|
|
|
|
|
|
def test_4xx_not_retried_fails_fast(monkeypatch):
|
|
c = _make_client(max_attempts=3)
|
|
_patch_sleep(monkeypatch)
|
|
calls = {"n": 0}
|
|
|
|
def bad_model(**kwargs):
|
|
calls["n"] += 1
|
|
raise APIStatusError(
|
|
message="model not found",
|
|
response=httpx.Response(404, request="req"),
|
|
body=None,
|
|
)
|
|
|
|
c.client.chat.completions.create = bad_model
|
|
with pytest.raises(LLMError):
|
|
c.complete("sys", "user")
|
|
# A 4xx is not transient: exactly one attempt, no retry.
|
|
assert calls["n"] == 1
|
|
|
|
|
|
def test_success_no_retry(monkeypatch):
|
|
c = _make_client(max_attempts=3)
|
|
_patch_sleep(monkeypatch)
|
|
calls = {"n": 0}
|
|
|
|
def ok(**kwargs):
|
|
calls["n"] += 1
|
|
return _resp()
|
|
|
|
c.client.chat.completions.create = ok
|
|
assert c.complete("sys", "user") == "ok"
|
|
assert calls["n"] == 1
|