Backlog alarm: only count new failures (last 24h)

FAILED/OCR FAIL rows never disappear on their own, so counting all of them
made the alarm fire every 5 minutes forever. Only stuck entries and
failures from the last 24 h trigger it now; older ones stay in the DB for
tools/gundi_requeue.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-10-07 20:04:46 +02:00
co-authored by Claude Opus 5.5
parent 6d518b4542
commit e7e5b6fb87
3 changed files with 48 additions and 6 deletions
+8 -5
View File
@@ -734,7 +734,10 @@ class Pipeline:
self.check_backlog()
def check_backlog(self) -> None:
"""Meldet haengende/gescheiterte Eintraege als [GUNDI][ALARM] (hoechstens alle 5 min)."""
"""Meldet haengende Eintraege und NEUE Fehlschlaege (letzte 24 h) als [GUNDI][ALARM], hoechstens alle 5 min.
Aeltere FAILED/FAIL bleiben in der DB (gundi_requeue), loesen aber keinen Dauer-Alarm aus.
"""
limit = self.gcfg.alarm_after_seconds
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute(
@@ -744,7 +747,7 @@ class Pipeline:
AND updated_at < now() - make_interval(secs => %s)),
count(*) FILTER (WHERE state = 'AWAITING_ATTACHMENT'
AND received_at < now() - make_interval(secs => %s)),
count(*) FILTER (WHERE state = 'FAILED')
count(*) FILTER (WHERE state = 'FAILED' AND updated_at > now() - interval '24 hours')
FROM remote_cam.gundi_inbox
WHERE received_at > now() - interval '14 days'
""",
@@ -756,7 +759,7 @@ class Pipeline:
SELECT
count(*) FILTER (WHERE ocr_state = 'PENDING'
AND updated_at < now() - make_interval(secs => %s)),
count(*) FILTER (WHERE ocr_state = 'FAIL')
count(*) FILTER (WHERE ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours')
FROM remote_cam.image
WHERE source = 'gundi' AND servertime > now() - interval '14 days'
""",
@@ -771,7 +774,7 @@ class Pipeline:
self._last_alarm = now
print(
f"[GUNDI][ALARM] backlog received_haengt={received_stuck} awaiting_attachment_alt={awaiting_old} "
f"failed={failed} ocr_haengt={ocr_stuck} ocr_fail={ocr_failed} (Schwelle {limit}s, 14 Tage) "
"- bei failed/ocr_fail: tools/gundi_requeue.py",
f"failed_24h={failed} ocr_haengt={ocr_stuck} ocr_fail_24h={ocr_failed} (haengt = laenger als {limit}s) "
"- Details/Zuruecksetzen: tools/gundi_requeue.py",
flush=True,
)