fix bd
This commit is contained in:
@@ -1,8 +1,15 @@
|
||||
"""Worker entrypoint: runs trigger checker + future background jobs."""
|
||||
"""Worker entrypoint: runs trigger checker + future background jobs.
|
||||
|
||||
Waits for the database (and required tables) to be ready before starting
|
||||
the background loops. This prevents the worker from crashing when the
|
||||
backend hasn't finished running `init_db()` yet (typical docker-compose
|
||||
race condition where both services start in parallel).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from app.db_wait import wait_for_db_or_exit
|
||||
from app.logging_setup import get_logger, setup_logging
|
||||
from app.workers.trigger_runner import main_loop as trigger_loop
|
||||
|
||||
@@ -12,6 +19,13 @@ log = get_logger("worker")
|
||||
async def main():
|
||||
setup_logging()
|
||||
log.info("worker_starting")
|
||||
|
||||
# Wait until DB is reachable and the `settings` table exists.
|
||||
# The backend's lifespan runs `init_db()` which creates tables; if the
|
||||
# worker comes up first, this loop will retry until that completes.
|
||||
await wait_for_db_or_exit(max_retries=60, delay=2.0)
|
||||
|
||||
log.info("worker_db_ready_starting_loops")
|
||||
# Run all background loops concurrently
|
||||
await asyncio.gather(
|
||||
trigger_loop(),
|
||||
|
||||
@@ -155,21 +155,34 @@ async def _fire_trigger(db, trigger: DeferredTrigger, session: Session, world: W
|
||||
|
||||
|
||||
async def main_loop():
|
||||
"""Main worker loop. Polls every N seconds for due triggers."""
|
||||
"""Main worker loop. Polls every N seconds for due triggers.
|
||||
|
||||
Resilient to transient DB errors: any error inside an iteration is logged
|
||||
and the loop sleeps for a fallback interval before retrying, instead of
|
||||
crashing the worker process.
|
||||
"""
|
||||
setup_logging()
|
||||
log.info("trigger_worker_started")
|
||||
fallback_interval = 30 # used when settings table is unreadable
|
||||
while True:
|
||||
interval = fallback_interval
|
||||
try:
|
||||
async with AsyncSessionLocal() as db:
|
||||
enabled = await _get_setting(db, "triggers.enabled", True)
|
||||
interval = int(await _get_setting(db, "triggers.check_interval", 30))
|
||||
interval = int(await _get_setting(db, "triggers.check_interval", fallback_interval))
|
||||
if enabled:
|
||||
fired = await check_and_fire_triggers()
|
||||
if fired:
|
||||
log.info("triggers_fired", count=fired)
|
||||
except Exception as e:
|
||||
log.error("trigger_worker_iteration_failed", error=str(e))
|
||||
await asyncio.sleep(max(5, int(await _get_setting_sleep())))
|
||||
log.error("trigger_worker_iteration_failed", error=f"{type(e).__name__}: {e}")
|
||||
# Always sleep with a safe positive interval; never let a DB error
|
||||
# escape this loop and crash the worker.
|
||||
try:
|
||||
sleep_for = max(5, int(interval))
|
||||
except Exception:
|
||||
sleep_for = fallback_interval
|
||||
await asyncio.sleep(sleep_for)
|
||||
|
||||
|
||||
async def _get_setting(db, key: str, default):
|
||||
|
||||
Reference in New Issue
Block a user