Backlog alarm: only count new failures (last 24h)

FAILED/OCR FAIL rows never disappear on their own, so counting all of them
made the alarm fire every 5 minutes forever. Only stuck entries and
failures from the last 24 h trigger it now; older ones stay in the DB for
tools/gundi_requeue.py.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-10-07 20:04:46 +02:00
co-authored by Claude Opus 5.5
parent 6d518b4542
commit e7e5b6fb87
3 changed files with 48 additions and 6 deletions
+1 -1
View File
@@ -99,7 +99,7 @@ Events ohne Bild nach `abandon_after_hours` als `ABANDONED`.
| Bild-Upload (`.../attachments/`) | S3-Zugriff mit kurzen Zeitlimits (`gundi.upload_*`: 5 s Connect, 30 s Read, 2 Versuche), ohne offene DB-Sperre. Bei S3-Stoerung `503` + `Retry-After: 30`; das Event bleibt bis `abandon_after_hours` auf `AWAITING_ATTACHMENT`, der Client kann wiederholen |
| Ingest/OCR im Hintergrund | S3-/DB-Stoerungen (`main.is_transient_error`) zaehlen **nicht** gegen `max_attempts`; der Sweep wiederholt alle `stale_after_seconds`. Nur echte Datenfehler fuehren zu `FAILED`/`FAIL` |
| Sweep | reiht dieselbe ID nicht doppelt ein (kein Stau bei langem Ausfall) |
| Alarm | `[GUNDI][ALARM] backlog ...` im Log (hoechstens alle 5 min), wenn Eintraege laenger als `gundi.alarm_after_seconds` in `RECEIVED`/`AWAITING_ATTACHMENT`/OCR-`PENDING` haengen oder `FAILED`/`FAIL` existieren |
| Alarm | `[GUNDI][ALARM] backlog ...` im Log (hoechstens alle 5 min), wenn Eintraege laenger als `gundi.alarm_after_seconds` in `RECEIVED`/`AWAITING_ATTACHMENT`/OCR-`PENDING` haengen oder in den letzten 24 h neue `FAILED`/`FAIL` entstanden sind (aeltere loesen keinen Dauer-Alarm aus) |
Gescheiterte Eintraege zuruecksetzen (Trockenlauf ohne `--apply`):
+8 -5
View File
@@ -734,7 +734,10 @@ class Pipeline:
self.check_backlog()
def check_backlog(self) -> None:
"""Meldet haengende/gescheiterte Eintraege als [GUNDI][ALARM] (hoechstens alle 5 min)."""
"""Meldet haengende Eintraege und NEUE Fehlschlaege (letzte 24 h) als [GUNDI][ALARM], hoechstens alle 5 min.
Aeltere FAILED/FAIL bleiben in der DB (gundi_requeue), loesen aber keinen Dauer-Alarm aus.
"""
limit = self.gcfg.alarm_after_seconds
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
cur.execute(
@@ -744,7 +747,7 @@ class Pipeline:
AND updated_at < now() - make_interval(secs => %s)),
count(*) FILTER (WHERE state = 'AWAITING_ATTACHMENT'
AND received_at < now() - make_interval(secs => %s)),
count(*) FILTER (WHERE state = 'FAILED')
count(*) FILTER (WHERE state = 'FAILED' AND updated_at > now() - interval '24 hours')
FROM remote_cam.gundi_inbox
WHERE received_at > now() - interval '14 days'
""",
@@ -756,7 +759,7 @@ class Pipeline:
SELECT
count(*) FILTER (WHERE ocr_state = 'PENDING'
AND updated_at < now() - make_interval(secs => %s)),
count(*) FILTER (WHERE ocr_state = 'FAIL')
count(*) FILTER (WHERE ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours')
FROM remote_cam.image
WHERE source = 'gundi' AND servertime > now() - interval '14 days'
""",
@@ -771,7 +774,7 @@ class Pipeline:
self._last_alarm = now
print(
f"[GUNDI][ALARM] backlog received_haengt={received_stuck} awaiting_attachment_alt={awaiting_old} "
f"failed={failed} ocr_haengt={ocr_stuck} ocr_fail={ocr_failed} (Schwelle {limit}s, 14 Tage) "
"- bei failed/ocr_fail: tools/gundi_requeue.py",
f"failed_24h={failed} ocr_haengt={ocr_stuck} ocr_fail_24h={ocr_failed} (haengt = laenger als {limit}s) "
"- Details/Zuruecksetzen: tools/gundi_requeue.py",
flush=True,
)
+39
View File
@@ -155,6 +155,45 @@ class FailureRecordingTests(unittest.TestCase):
self.assertIn("'FAIL'", sql)
class BacklogAlarmTests(unittest.TestCase):
def run_check(self, inbox_row, image_row, pipeline=None):
cur = FakeCursor(rows=[inbox_row, image_row])
p = pipeline or bare_pipeline()
p.gcfg = SimpleNamespace(alarm_after_seconds=1800)
p._last_alarm = 0.0
out = []
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(" ".join(map(str, a)))):
p.check_backlog()
return p, cur, out
def test_silent_when_nothing_is_wrong(self):
_, _, out = self.run_check((0, 0, 0), (0, 0))
self.assertEqual(out, [])
def test_alarm_for_stuck_and_new_failures(self):
_, _, out = self.run_check((2, 0, 1), (0, 3))
self.assertEqual(len(out), 1)
self.assertIn("[GUNDI][ALARM]", out[0])
self.assertIn("received_haengt=2", out[0])
self.assertIn("failed_24h=1", out[0])
self.assertIn("ocr_fail_24h=3", out[0])
def test_failures_older_than_24h_are_not_counted(self):
_, cur, _ = self.run_check((0, 0, 0), (0, 0))
sqls = " ".join(sql for sql, _ in cur.executed)
self.assertIn("state = 'FAILED' AND updated_at > now() - interval '24 hours'", sqls)
self.assertIn("ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours'", sqls)
def test_alarm_is_throttled(self):
p, _, first = self.run_check((1, 0, 0), (0, 0))
self.assertEqual(len(first), 1)
cur = FakeCursor(rows=[(1, 0, 0), (0, 0)])
out = []
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(a)):
p.check_backlog() # direkt danach -> unterdrueckt
self.assertEqual(out, [])
class SubmitDedupeTests(unittest.TestCase):
def test_same_id_is_queued_once_until_done(self):
p = bare_pipeline()