This commit is contained in:
Mikan
2026-06-19 11:30:38 +03:00
parent 53c89829a8
commit 59b6262c70
5 changed files with 160 additions and 38 deletions

View File

@@ -1,8 +1,15 @@
"""Worker entrypoint: runs trigger checker + future background jobs."""
"""Worker entrypoint: runs trigger checker + future background jobs.
Waits for the database (and required tables) to be ready before starting
the background loops. This prevents the worker from crashing when the
backend hasn't finished running `init_db()` yet (typical docker-compose
race condition where both services start in parallel).
"""
from __future__ import annotations
import asyncio
from app.db_wait import wait_for_db_or_exit
from app.logging_setup import get_logger, setup_logging
from app.workers.trigger_runner import main_loop as trigger_loop
@@ -12,6 +19,13 @@ log = get_logger("worker")
async def main():
setup_logging()
log.info("worker_starting")
# Wait until DB is reachable and the `settings` table exists.
# The backend's lifespan runs `init_db()` which creates tables; if the
# worker comes up first, this loop will retry until that completes.
await wait_for_db_or_exit(max_retries=60, delay=2.0)
log.info("worker_db_ready_starting_loops")
# Run all background loops concurrently
await asyncio.gather(
trigger_loop(),

View File

@@ -155,21 +155,34 @@ async def _fire_trigger(db, trigger: DeferredTrigger, session: Session, world: W
async def main_loop():
"""Main worker loop. Polls every N seconds for due triggers."""
"""Main worker loop. Polls every N seconds for due triggers.
Resilient to transient DB errors: any error inside an iteration is logged
and the loop sleeps for a fallback interval before retrying, instead of
crashing the worker process.
"""
setup_logging()
log.info("trigger_worker_started")
fallback_interval = 30 # used when settings table is unreadable
while True:
interval = fallback_interval
try:
async with AsyncSessionLocal() as db:
enabled = await _get_setting(db, "triggers.enabled", True)
interval = int(await _get_setting(db, "triggers.check_interval", 30))
interval = int(await _get_setting(db, "triggers.check_interval", fallback_interval))
if enabled:
fired = await check_and_fire_triggers()
if fired:
log.info("triggers_fired", count=fired)
except Exception as e:
log.error("trigger_worker_iteration_failed", error=str(e))
await asyncio.sleep(max(5, int(await _get_setting_sleep())))
log.error("trigger_worker_iteration_failed", error=f"{type(e).__name__}: {e}")
# Always sleep with a safe positive interval; never let a DB error
# escape this loop and crash the worker.
try:
sleep_for = max(5, int(interval))
except Exception:
sleep_for = fallback_interval
await asyncio.sleep(sleep_for)
async def _get_setting(db, key: str, default):