Backlog alarm: only count new failures (last 24h)
FAILED/OCR FAIL rows never disappear on their own, so counting all of them made the alarm fire every 5 minutes forever. Only stuck entries and failures from the last 24 h trigger it now; older ones stay in the DB for tools/gundi_requeue.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+8
-5
@@ -734,7 +734,10 @@ class Pipeline:
|
||||
self.check_backlog()
|
||||
|
||||
def check_backlog(self) -> None:
|
||||
"""Meldet haengende/gescheiterte Eintraege als [GUNDI][ALARM] (hoechstens alle 5 min)."""
|
||||
"""Meldet haengende Eintraege und NEUE Fehlschlaege (letzte 24 h) als [GUNDI][ALARM], hoechstens alle 5 min.
|
||||
|
||||
Aeltere FAILED/FAIL bleiben in der DB (gundi_requeue), loesen aber keinen Dauer-Alarm aus.
|
||||
"""
|
||||
limit = self.gcfg.alarm_after_seconds
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
@@ -744,7 +747,7 @@ class Pipeline:
|
||||
AND updated_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE state = 'AWAITING_ATTACHMENT'
|
||||
AND received_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE state = 'FAILED')
|
||||
count(*) FILTER (WHERE state = 'FAILED' AND updated_at > now() - interval '24 hours')
|
||||
FROM remote_cam.gundi_inbox
|
||||
WHERE received_at > now() - interval '14 days'
|
||||
""",
|
||||
@@ -756,7 +759,7 @@ class Pipeline:
|
||||
SELECT
|
||||
count(*) FILTER (WHERE ocr_state = 'PENDING'
|
||||
AND updated_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE ocr_state = 'FAIL')
|
||||
count(*) FILTER (WHERE ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours')
|
||||
FROM remote_cam.image
|
||||
WHERE source = 'gundi' AND servertime > now() - interval '14 days'
|
||||
""",
|
||||
@@ -771,7 +774,7 @@ class Pipeline:
|
||||
self._last_alarm = now
|
||||
print(
|
||||
f"[GUNDI][ALARM] backlog received_haengt={received_stuck} awaiting_attachment_alt={awaiting_old} "
|
||||
f"failed={failed} ocr_haengt={ocr_stuck} ocr_fail={ocr_failed} (Schwelle {limit}s, 14 Tage) "
|
||||
"- bei failed/ocr_fail: tools/gundi_requeue.py",
|
||||
f"failed_24h={failed} ocr_haengt={ocr_stuck} ocr_fail_24h={ocr_failed} (haengt = laenger als {limit}s) "
|
||||
"- Details/Zuruecksetzen: tools/gundi_requeue.py",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user