fix
This commit is contained in:
@@ -16,6 +16,37 @@ from app.models import LlmCallLog
|
||||
log = get_logger("llm")
|
||||
|
||||
|
||||
# vendor-specific end-of-sequence / control tokens that some local models
|
||||
# emit into the content stream. Strip them so they don't leak to the user.
|
||||
_EOS_TOKENS = (
|
||||
"<eos>",
|
||||
"</s>",
|
||||
"<|endoftext|>",
|
||||
"<|im_end|>",
|
||||
"<|end|>",
|
||||
"<|eot_id|>",
|
||||
"<|eom_id|>",
|
||||
)
|
||||
|
||||
|
||||
def _clean_model_text(text: str) -> str:
|
||||
"""Remove vendor-specific end-of-sequence tokens and collapse whitespace.
|
||||
|
||||
Some local models leak control tokens into the visible content stream.
|
||||
We strip them so they never reach the user.
|
||||
"""
|
||||
if not text:
|
||||
return text
|
||||
cleaned = text
|
||||
for tok in _EOS_TOKENS:
|
||||
cleaned = cleaned.replace(tok, "")
|
||||
while "\n\n\n" in cleaned:
|
||||
cleaned = cleaned.replace("\n\n\n", "\n\n")
|
||||
return cleaned
|
||||
|
||||
|
||||
|
||||
|
||||
class LlmResponse:
|
||||
"""Non-streaming response wrapper."""
|
||||
|
||||
@@ -93,7 +124,7 @@ class LlmClient:
|
||||
data = resp.json()
|
||||
choice = (data.get("choices") or [{}])[0]
|
||||
msg = choice.get("message", {})
|
||||
text = msg.get("content") or ""
|
||||
text = _clean_model_text(msg.get("content") or "")
|
||||
tool_calls = msg.get("tool_calls") or []
|
||||
usage = data.get("usage") or {}
|
||||
except httpx.ConnectError as e:
|
||||
@@ -206,8 +237,10 @@ class LlmClient:
|
||||
continue
|
||||
delta = choices[0].get("delta", {})
|
||||
if delta.get("content"):
|
||||
full_text_parts.append(delta["content"])
|
||||
yield {"type": "delta", "content": delta["content"]}
|
||||
piece = _clean_model_text(delta["content"])
|
||||
if piece:
|
||||
full_text_parts.append(piece)
|
||||
yield {"type": "delta", "content": piece}
|
||||
if delta.get("tool_calls"):
|
||||
for tc in delta["tool_calls"]:
|
||||
idx = tc.get("index", 0)
|
||||
|
||||
@@ -34,6 +34,8 @@ EDITABLE_SETTING_KEYS = {
|
||||
"embedding.model": str, # e.g. text-embedding-3-small, bge-m3, nomic-embed-text
|
||||
"embedding.dim": int, # vector dimension; 0 = auto-probe from endpoint
|
||||
"embedding.request_timeout": int, # request timeout, seconds
|
||||
# UI customization (logo URL/path shown in navbar + home page + favicon)
|
||||
"ui.logo_url": str, # e.g. "/logo.png", "https://.../logo.png", or "data:image/png;base64,..."
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user