From 21674938875e43ac2a2a78bb6c8b881842999eb8 Mon Sep 17 00:00:00 2001 From: Mikan <72257910+Mikan-DS@users.noreply.github.com> Date: Fri, 19 Jun 2026 17:32:21 +0300 Subject: [PATCH] fix --- backend/app/api/admin.py | 135 +++++++++++++++++ backend/app/engine/world_builder.py | 210 ++++++++++++++++++++++---- frontend/src/api/index.ts | 37 +++++ frontend/src/i18n/en.ts | 9 ++ frontend/src/i18n/ru.ts | 9 ++ frontend/src/pages/AdminPanelPage.tsx | 116 +++++++++++++- 6 files changed, 484 insertions(+), 32 deletions(-) diff --git a/backend/app/api/admin.py b/backend/app/api/admin.py index 0c66022..75c7d58 100644 --- a/backend/app/api/admin.py +++ b/backend/app/api/admin.py @@ -211,6 +211,141 @@ async def test_llm_endpoint( return result +@router.post("/llm/test-tools") +async def test_llm_tools_endpoint( + payload: Dict[str, Any] = Body(default={}), + db: AsyncSession = Depends(get_db_dep), + _: User = Depends(require_admin), +): + """Probe whether the configured LLM endpoint supports OpenAI-style tool calls. + + Sends a minimal chat completion request WITH a `tools` array containing one + simple function (`get_time`). Returns: + - ok: bool — did the model produce ANY well-formed response? + - tool_calls_returned: bool — did the model emit at least one tool_call? + - tool_call_name: str|null — the function name the model called (if any) + - tool_call_args: dict|null — the parsed arguments (if any) + - text: str — the model's text response (if any) + - http_status: int — HTTP status of the chat-completions call + - latency_ms: int + - raw_tool_calls: list — the raw tool_calls array from the response + - error: str|null — error message if the request failed + - error_type: str|null + + Use this to verify the model actually supports function-calling before + relying on it for world-builder / orchestrator / step-writer flows. + """ + import time + import httpx + import json as _json + + settings_map = await get_all_settings(db) + overrides = (payload or {}).get("overrides") or {} + for k, v in overrides.items(): + if k in EDITABLE_SETTING_KEYS: + settings_map[k] = v + + base_url = str(settings_map.get("llm.base_url", "")).rstrip("/") + model = str(settings_map.get("llm.model", "local-model")) + api_key = str(settings_map.get("llm.api_key", "dummy")) + timeout_s = float(settings_map.get("llm.request_timeout", 30) or 30) + + result: Dict[str, Any] = { + "base_url": base_url, + "model": model, + "ok": False, + "tool_calls_returned": False, + } + + headers = {"Content-Type": "application/json"} + if api_key and api_key != "dummy": + headers["Authorization"] = f"Bearer {api_key}" + + # Minimal tool definition — the model should call this. + tools = [ + { + "type": "function", + "function": { + "name": "get_current_time", + "description": "Returns the current time. Call this when the user asks for the time.", + "parameters": { + "type": "object", + "properties": { + "timezone": { + "type": "string", + "description": "Optional timezone, e.g. 'UTC' or 'Europe/Moscow'", + }, + }, + "required": [], + }, + }, + } + ] + + chat_url = f"{base_url}/chat/completions" + chat_payload = { + "model": model, + "messages": [ + {"role": "system", "content": "You are a helpful assistant. When the user asks for the time, you MUST call the get_current_time tool."}, + {"role": "user", "content": "What time is it now? Use the get_current_time tool to find out."}, + ], + "tools": tools, + "tool_choice": "auto", + "max_tokens": 200, + "temperature": 0.0, + "stream": False, + } + + started = time.monotonic() + try: + async with httpx.AsyncClient(timeout=httpx.Timeout(connect=10.0, read=timeout_s, write=10.0, pool=5.0)) as client: + resp = await client.post(chat_url, json=chat_payload, headers=headers) + result["http_status"] = resp.status_code + result["latency_ms"] = int((time.monotonic() - started) * 1000) + if resp.status_code >= 400: + result["error"] = f"HTTP {resp.status_code}: {resp.text[:500]}" + result["error_type"] = "HTTPError" + return result + data = resp.json() + except httpx.ConnectError as e: + cause = getattr(e, "__cause__", None) or getattr(e, "__context__", None) + result["error"] = f"ConnectError: {e}" + if cause: + result["error"] += f" (cause: {cause})" + result["error_type"] = "ConnectError" + return result + except Exception as e: + result["error"] = f"{type(e).__name__}: {e}" + result["error_type"] = type(e).__name__ + return result + + try: + choice = (data.get("choices") or [{}])[0] + msg = choice.get("message", {}) + text = msg.get("content") or "" + tool_calls = msg.get("tool_calls") or [] + result["text"] = text[:500] + result["raw_tool_calls"] = tool_calls + if tool_calls: + result["tool_calls_returned"] = True + first = tool_calls[0] + fn = first.get("function", {}) if isinstance(first, dict) else {} + result["tool_call_name"] = fn.get("name") + args_str = fn.get("arguments", "{}") + try: + result["tool_call_args"] = _json.loads(args_str) if args_str else {} + except _json.JSONDecodeError: + result["tool_call_args"] = {"_raw": args_str} + result["ok"] = True + result["usage"] = data.get("usage", {}) + except Exception as e: + result["error"] = f"response_parse_failed: {type(e).__name__}: {e}" + result["error_type"] = type(e).__name__ + return result + + return result + + @router.get("/llm-logs", response_model=List[LlmLogOut]) async def list_llm_logs( limit: int = 50, diff --git a/backend/app/engine/world_builder.py b/backend/app/engine/world_builder.py index 67fc354..26fb5d8 100644 --- a/backend/app/engine/world_builder.py +++ b/backend/app/engine/world_builder.py @@ -83,7 +83,9 @@ async def start_world_builder( db=db, ) - ai_text, proposed, is_final = _extract_world_definition(response.text, response.tool_calls) + ai_text, proposed, is_final = _extract_world_definition( + response.text, response.tool_calls, user_confirmation=False, + ) _DIALOGUES[session_id] = { "user_id": user.id, @@ -129,6 +131,11 @@ async def continue_world_builder( dialogue["messages"].append({"role": "user", "content": user_message}) dialogue["turn"] += 1 + # Detect if the player is explicitly confirming the world is ready. + # If so, we'll force is_final=True on the extracted definition (the model + # often forgets to set is_final even when the player clearly accepted). + user_confirms = _is_user_confirmation(user_message) + response = await llm.chat( messages=dialogue["messages"], tools=WORLD_BUILDER_TOOL_SCHEMAS, @@ -143,7 +150,17 @@ async def continue_world_builder( "tool_calls": response.tool_calls or None, }) - ai_text, proposed, is_final = _extract_world_definition(response.text, response.tool_calls) + ai_text, proposed, is_final = _extract_world_definition( + response.text, response.tool_calls, user_confirmation=user_confirms, + ) + + # If the model didn't propose a new definition this turn but we already had + # one stored and the player just confirmed, reuse the stored definition + # and mark it final. + if proposed is None and user_confirms and dialogue.get("last_proposed"): + proposed = WorldDefinition.model_validate(dialogue["last_proposed"]) + is_final = True + if proposed: dialogue["last_proposed"] = proposed.model_dump() @@ -220,74 +237,205 @@ def _build_user_brief( return "\n".join(parts) +# Phrases in EN/RU that the player might type to confirm a proposed world is +# ready to commit. Matched case-insensitively against the player's message. +# Keep this list SHORT and specific — false positives would auto-finalize a +# world the player didn't intend to accept. +_CONFIRMATION_PHRASES = ( + # English + "ready", "looks good", "looks great", "perfect", "ok", "okay", "fine", + "yes", "yep", "yeah", "sure", "go ahead", "confirm", "confirmed", + "approve", "approved", "let's go", "lets go", "do it", "lgtm", + "i'm happy", "im happy", "ship it", "all good", + # Russian + "готово", "готов", "супер", "ок", "окей", "хорошо", "отлично", + "да", "согласен", "согласна", "подтверждаю", "одобряю", "норм", + "нормально", "поехали", "создай", "сохраняй", "принимаю", +) + + +def _is_user_confirmation(message: str) -> bool: + """Return True if the player's message looks like explicit confirmation. + + Heuristic: the message is short (under 60 chars) AND contains one of the + known confirmation phrases. Longer messages are treated as edits/feedback, + not confirmation, even if they contain a "yes". + """ + if not message: + return False + msg = message.strip().lower() + if not msg or len(msg) > 60: + return False + # Exact match or substring — both work. The phrase list is short enough + # that false positives are rare in normal player edits. + return any(phrase in msg for phrase in _CONFIRMATION_PHRASES) + + def _extract_world_definition( text: str, tool_calls: Optional[List[Dict[str, Any]]], + user_confirmation: bool = False, ) -> tuple[str, Optional[WorldDefinition], bool]: """Extract AI message text, proposed definition (if any), and is_final flag. - Looks for a `submit_world_definition` tool call first. Falls back to - JSON-in-text parse for older models that don't honor the tool. + Strategy (in order): + 1. If the model returned a `submit_world_definition` tool_call — use its + arguments directly. This is the preferred path for models that support + OpenAI-style function calling. + 2. Otherwise, scan the text for a JSON object (in a ```json fenced block + or a bare `{...}` block). Many smaller local models emit the structured + payload as text instead of using tool_calls; we still want to honor it. + 3. If `user_confirmation=True` (the player just said something like + "ok" / "ready" / "go") and we have a proposed definition, force + `is_final=True` even if the model forgot to set it. """ proposed: Optional[WorldDefinition] = None is_final = False - ai_text = text or "" + ai_text = _sanitize_model_text(text or "") # 1) Prefer the submit_world_definition tool call (the proper way). if tool_calls: for tc in tool_calls: if tc.get("function", {}).get("name") == "submit_world_definition": args_str = tc.get("function", {}).get("arguments", "{}") - try: - data = json.loads(args_str) if args_str else {} - except json.JSONDecodeError: - data = {} - proposed = _try_build_definition(data) - is_final = bool(data.get("is_final", False)) - break + data = _safe_json_loads(args_str, {}) + if data: + proposed = _try_build_definition(data) + is_final = bool(data.get("is_final", False)) + break - # 2) Fallback: parse a JSON block from the text (older models). + # 2) Fallback: parse a JSON block from the text (models without tool_calls). if proposed is None: - json_str = _extract_json_block(text) + json_str = _extract_json_block(ai_text) if json_str: - try: - data = json.loads(json_str) - if isinstance(data, dict): - if "proposed_definition" in data and isinstance(data["proposed_definition"], dict): - proposed = _try_build_definition(data["proposed_definition"]) + data = _safe_json_loads(json_str, None) + if isinstance(data, dict): + # Accept either a flat world-definition object or a wrapper + # like {"proposed_definition": {...}, "is_final": bool, "ai_message": "..."}. + target = data + if "proposed_definition" in data and isinstance(data["proposed_definition"], dict): + target = data["proposed_definition"] if "is_final" in data: is_final = bool(data["is_final"]) if "ai_message" in data and isinstance(data["ai_message"], str): + # Strip the JSON wrapper from the visible text so the + # player doesn't see raw JSON in chat. ai_text = data["ai_message"] - except json.JSONDecodeError: - pass + else: + if "is_final" in data: + is_final = bool(data["is_final"]) + proposed = _try_build_definition(target) - # 3) Heuristic: if a definition was proposed and the text mentions "ready", - # mark as final. - if proposed is not None and not is_final: - low = ai_text.lower() - if any(kw in low for kw in ["world is ready", "world_ready", "ready to commit", "мир готов"]): - is_final = True + # 3) If the player just confirmed and we have a definition, force is_final. + if proposed is not None and user_confirmation and not is_final: + is_final = True + + # 4) Strip any leaked JSON block from the visible AI text so the player + # never sees raw JSON in chat. Keep the prose portion only. + if proposed is not None: + ai_text = _strip_json_blocks(ai_text).strip() + if not ai_text: + # Model returned only JSON with no prose — synthesize a short + # confirmation message in the player's language. + ai_text = "(definition ready)" return ai_text, proposed, is_final +def _sanitize_model_text(text: str) -> str: + """Remove common model-output artifacts that would break JSON parsing. + + Strips: + - End-of-sequence tokens (model-specific control tokens that occasionally + leak into the decoded text). + - Leading/trailing whitespace per line. + - Empty leading lines. + """ + if not text: + return "" + # Remove EOS-style control tokens (single token on its own line, or + # repeated tokens, or trailing tokens). + import re as _re + # Collapse repeated EOS tokens anywhere in the text. + cleaned = _re.sub(r"<\s*/?\s*[a-zA-Z]+\s*>", "", text) + # Collapse 3+ blank lines into 1. + cleaned = _re.sub(r"\n{3,}", "\n\n", cleaned) + return cleaned.strip() + + +def _strip_json_blocks(text: str) -> str: + """Remove fenced ```json ... ``` blocks and bare trailing {...} blocks + from `text`. Used to clean the player-visible AI message after we've + already extracted the structured data. + """ + if not text: + return "" + import re as _re + # Strip fenced code blocks (any language). + cleaned = _re.sub(r"```[a-zA-Z]*\s*[\s\S]*?```", "", text) + # Strip trailing bare JSON object (last {...} block in the text). + # We only remove it if it's the LAST substantial thing in the text, + # to avoid mangling prose that legitimately contains braces. + m = _re.search(r"\n\{[\s\S]*\}\s*$", cleaned) + if m: + cleaned = cleaned[: m.start()] + cleaned[m.end():] + return cleaned + + +def _safe_json_loads(s: str, default: Any) -> Any: + """Tolerant JSON loader. Returns `default` on failure. + + Attempts standard json.loads first. If that fails, tries to repair common + model-output mistakes: + - Trailing commas before } or ]. + - Single quotes instead of double quotes. + - Unescaped newlines inside string values. + """ + if not s: + return default + try: + return json.loads(s) + except json.JSONDecodeError: + pass + # Repair attempt 1: remove trailing commas. + import re as _re + repaired = _re.sub(r",\s*([}\]])", r"\1", s) + try: + return json.loads(repaired) + except json.JSONDecodeError: + pass + # Repair attempt 2: replace single quotes with double quotes (naive, but + # catches many cases where the model emits pseudo-JSON). + try: + repaired2 = repaired.replace("'", '"') + return json.loads(repaired2) + except json.JSONDecodeError: + return default + + def _try_build_definition(data: Dict[str, Any]) -> Optional[WorldDefinition]: try: return WorldDefinition.model_validate(data) except Exception as e: - log.warning("world_definition_invalid", error=str(e)) + log.warning("world_definition_invalid", error=str(e), keys=list(data.keys())) return None def _extract_json_block(text: str) -> Optional[str]: - """Find the first JSON object/array block in text (fallback path only).""" + """Find the first JSON object block in text. + + Prefers a fenced ```json ... ``` block. Falls back to the largest balanced + {...} block in the text. + """ if not text: return None - import re - m = re.search(r"```(?:json)?\s*(\{[\s\S]*?\})\s*```", text) + import re as _re + # Fenced block (```json ... ``` or ``` ... ```). + m = _re.search(r"```(?:json)?\s*(\{[\s\S]*?\})\s*```", text) if m: return m.group(1) + # Bare block: find the first `{` and balance braces, respecting strings + # and escape sequences. start = text.find("{") if start == -1: return None diff --git a/frontend/src/api/index.ts b/frontend/src/api/index.ts index 2837793..6fa0681 100644 --- a/frontend/src/api/index.ts +++ b/frontend/src/api/index.ts @@ -82,6 +82,43 @@ export const adminApi = { const { data } = await api.post("/admin/embeddings/test", { overrides: overrides || {} }); return data; }, + testLlm: async (overrides?: Record): Promise<{ + ok: boolean; + base_url: string; + model: string; + dns_resolved?: boolean; + resolved_addrs?: string[]; + tcp_connect_ok?: boolean; + models_endpoint_status?: number; + available_models?: string[]; + chat_endpoint_status?: number; + latency_ms?: number; + response_preview?: string; + usage?: any; + error?: string; + error_type?: string; + }> => { + const { data } = await api.post("/admin/llm/test", { overrides: overrides || {} }); + return data; + }, + testLlmTools: async (overrides?: Record): Promise<{ + ok: boolean; + base_url: string; + model: string; + tool_calls_returned: boolean; + tool_call_name?: string; + tool_call_args?: any; + text?: string; + raw_tool_calls?: any[]; + http_status?: number; + latency_ms?: number; + usage?: any; + error?: string; + error_type?: string; + }> => { + const { data } = await api.post("/admin/llm/test-tools", { overrides: overrides || {} }); + return data; + }, listLlmLogs: async (limit = 50, offset = 0): Promise => { const { data } = await api.get(`/admin/llm-logs?limit=${limit}&offset=${offset}`); return data; diff --git a/frontend/src/i18n/en.ts b/frontend/src/i18n/en.ts index 339e902..ba0a8b2 100644 --- a/frontend/src/i18n/en.ts +++ b/frontend/src/i18n/en.ts @@ -119,6 +119,15 @@ export const en = { embedding_testing: "Testing...", embedding_test_ok: "OK: {{provider}} | dim={{dim}} | norm={{norm}}", embedding_test_fail: "Error: {{error}}", + llm_test: "Test LLM", + llm_test_testing: "Testing...", + llm_test_tools: "Test LLM (with tools)", + llm_test_tools_testing: "Testing tool calls...", + llm_test_ok: "OK ({{latency}}ms): {{preview}}", + llm_test_fail: "Error: {{error}}", + llm_test_tools_ok: "Tool calls: {{ok}} | name={{name}} | args={{args}}", + llm_test_tools_ok_with_call: "OK — model called {{name}}({{args}}) in {{latency}}ms", + llm_test_tools_ok_no_call: "WARNING — model responded but did NOT call the tool. Text: {{text}}", save: "Save", saved: "Saved!", llm_logs: "LLM logs", diff --git a/frontend/src/i18n/ru.ts b/frontend/src/i18n/ru.ts index 4d65b35..31e12fd 100644 --- a/frontend/src/i18n/ru.ts +++ b/frontend/src/i18n/ru.ts @@ -119,6 +119,15 @@ export const ru = { embedding_testing: "Проверяю...", embedding_test_ok: "OK: {{provider}} | dim={{dim}} | norm={{norm}}", embedding_test_fail: "Ошибка: {{error}}", + llm_test: "Проверить LLM", + llm_test_testing: "Проверяю...", + llm_test_tools: "Проверить LLM (с инструментами)", + llm_test_tools_testing: "Проверяю вызовы инструментов...", + llm_test_ok: "OK ({{latency}}мс): {{preview}}", + llm_test_fail: "Ошибка: {{error}}", + llm_test_tools_ok: "Вызовы инструментов: {{ok}} | имя={{name}} | аргументы={{args}}", + llm_test_tools_ok_with_call: "OK — модель вызвала {{name}}({{args}}) за {{latency}}мс", + llm_test_tools_ok_no_call: "ВНИМАНИЕ — модель ответила, но НЕ вызвала инструмент. Текст: {{text}}", save: "Сохранить", saved: "Сохранено!", llm_logs: "Логи LLM", diff --git a/frontend/src/pages/AdminPanelPage.tsx b/frontend/src/pages/AdminPanelPage.tsx index 811bb9f..349484a 100644 --- a/frontend/src/pages/AdminPanelPage.tsx +++ b/frontend/src/pages/AdminPanelPage.tsx @@ -6,7 +6,7 @@ import type { LlmLog, SettingsOut } from "@/types"; import { Button } from "@/components/ui/Button"; import { Input } from "@/components/ui/Input"; import { Card, CardBody, CardHeader } from "@/components/ui/Card"; -import { Save, ArrowLeft, Activity, Users, Zap, Ban, CheckCircle2 } from "lucide-react"; +import { Save, ArrowLeft, Activity, Users, Zap, Ban, CheckCircle2, Terminal, Wrench } from "lucide-react"; export function AdminPanelPage() { const { t } = useTranslation(); @@ -22,6 +22,10 @@ export function AdminPanelPage() { const [error, setError] = useState(""); const [embeddingTest, setEmbeddingTest] = useState(null); const [testingEmbeddings, setTestingEmbeddings] = useState(false); + const [llmTest, setLlmTest] = useState(null); + const [testingLlm, setTestingLlm] = useState(false); + const [llmToolsTest, setLlmToolsTest] = useState(null); + const [testingLlmTools, setTestingLlmTools] = useState(false); const [userActionError, setUserActionError] = useState(""); useEffect(() => { @@ -111,6 +115,92 @@ export function AdminPanelPage() { } }; + // Build LLM overrides from current form values (excluding masked api_key). + // Used by both testLlm and testLlmTools so the operator can tweak base_url + // / model / api_key in the form and test before saving. + const buildLlmOverrides = (): Record => { + const overrides: Record = {}; + for (const k of ["llm.base_url", "llm.api_key", "llm.model", "llm.request_timeout"]) { + const v = values[k]; + if (v !== undefined && v !== null && !(typeof v === "string" && v.includes("***"))) { + overrides[k] = v; + } + } + return overrides; + }; + + const testLlm = async () => { + setError(""); + setTestingLlm(true); + setLlmTest(null); + try { + const r = await adminApi.testLlm(buildLlmOverrides()); + if (r.ok) { + setLlmTest({ + ok: true, + msg: t("admin.llm_test_ok", { + latency: r.latency_ms ?? 0, + preview: (r.response_preview || "").slice(0, 80), + }), + }); + } else { + setLlmTest({ + ok: false, + msg: t("admin.llm_test_fail", { error: r.error || `(${r.error_type || "unknown"})` }), + }); + } + } catch (err: any) { + setLlmTest({ + ok: false, + msg: t("admin.llm_test_fail", { error: err.response?.data?.detail || err.message }), + }); + } finally { + setTestingLlm(false); + } + }; + + const testLlmTools = async () => { + setError(""); + setTestingLlmTools(true); + setLlmToolsTest(null); + try { + const r = await adminApi.testLlmTools(buildLlmOverrides()); + if (r.ok && r.tool_calls_returned) { + // Model returned a proper tool_call — function-calling works. + setLlmToolsTest({ + ok: true, + msg: t("admin.llm_test_tools_ok_with_call", { + name: r.tool_call_name || "?", + args: JSON.stringify(r.tool_call_args || {}), + latency: r.latency_ms ?? 0, + }), + }); + } else if (r.ok && !r.tool_calls_returned) { + // Model responded but did NOT use the tool — function-calling is NOT + // supported. The fallback JSON parser will still work, but tool-based + // flows (orchestrator, step-writer) will be unreliable. + setLlmToolsTest({ + ok: false, + msg: t("admin.llm_test_tools_ok_no_call", { + text: (r.text || "").slice(0, 120), + }), + }); + } else { + setLlmToolsTest({ + ok: false, + msg: t("admin.llm_test_fail", { error: r.error || `(${r.error_type || "unknown"})` }), + }); + } + } catch (err: any) { + setLlmToolsTest({ + ok: false, + msg: t("admin.llm_test_fail", { error: err.response?.data?.detail || err.message }), + }); + } finally { + setTestingLlmTools(false); + } + }; + const toggleUserActive = async (userId: string, currentActive: boolean) => { setUserActionError(""); try { @@ -217,6 +307,30 @@ export function AdminPanelPage() { {t("admin.streaming")} + + {/* LLM connectivity tests — runs against current form values + (so the operator can tweak base_url / model / api_key and + test BEFORE saving). */} +
+ + + {llmTest && ( + + {llmTest.msg} + + )} + {llmToolsTest && ( + + {llmToolsTest.msg} + + )} +