From e7e5b6fb8703a4d766715aa90e637bcfe30407b8 Mon Sep 17 00:00:00 2001 From: Dominik Dachs Date: Wed, 7 Oct 2026 20:04:46 +0200 Subject: [PATCH] Backlog alarm: only count new failures (last 24h) FAILED/OCR FAIL rows never disappear on their own, so counting all of them made the alarm fire every 5 minutes forever. Only stuck entries and failures from the last 24 h trigger it now; older ones stay in the DB for tools/gundi_requeue.py. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- gundi_pipeline.py | 13 +++++++----- tests/test_gundi_resilience.py | 39 ++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b0228be..858d465 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Events ohne Bild nach `abandon_after_hours` als `ABANDONED`. | Bild-Upload (`.../attachments/`) | S3-Zugriff mit kurzen Zeitlimits (`gundi.upload_*`: 5 s Connect, 30 s Read, 2 Versuche), ohne offene DB-Sperre. Bei S3-Stoerung `503` + `Retry-After: 30`; das Event bleibt bis `abandon_after_hours` auf `AWAITING_ATTACHMENT`, der Client kann wiederholen | | Ingest/OCR im Hintergrund | S3-/DB-Stoerungen (`main.is_transient_error`) zaehlen **nicht** gegen `max_attempts`; der Sweep wiederholt alle `stale_after_seconds`. Nur echte Datenfehler fuehren zu `FAILED`/`FAIL` | | Sweep | reiht dieselbe ID nicht doppelt ein (kein Stau bei langem Ausfall) | -| Alarm | `[GUNDI][ALARM] backlog ...` im Log (hoechstens alle 5 min), wenn Eintraege laenger als `gundi.alarm_after_seconds` in `RECEIVED`/`AWAITING_ATTACHMENT`/OCR-`PENDING` haengen oder `FAILED`/`FAIL` existieren | +| Alarm | `[GUNDI][ALARM] backlog ...` im Log (hoechstens alle 5 min), wenn Eintraege laenger als `gundi.alarm_after_seconds` in `RECEIVED`/`AWAITING_ATTACHMENT`/OCR-`PENDING` haengen oder in den letzten 24 h neue `FAILED`/`FAIL` entstanden sind (aeltere loesen keinen Dauer-Alarm aus) | Gescheiterte Eintraege zuruecksetzen (Trockenlauf ohne `--apply`): diff --git a/gundi_pipeline.py b/gundi_pipeline.py index debff70..d15e310 100644 --- a/gundi_pipeline.py +++ b/gundi_pipeline.py @@ -734,7 +734,10 @@ class Pipeline: self.check_backlog() def check_backlog(self) -> None: - """Meldet haengende/gescheiterte Eintraege als [GUNDI][ALARM] (hoechstens alle 5 min).""" + """Meldet haengende Eintraege und NEUE Fehlschlaege (letzte 24 h) als [GUNDI][ALARM], hoechstens alle 5 min. + + Aeltere FAILED/FAIL bleiben in der DB (gundi_requeue), loesen aber keinen Dauer-Alarm aus. + """ limit = self.gcfg.alarm_after_seconds with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur: cur.execute( @@ -744,7 +747,7 @@ class Pipeline: AND updated_at < now() - make_interval(secs => %s)), count(*) FILTER (WHERE state = 'AWAITING_ATTACHMENT' AND received_at < now() - make_interval(secs => %s)), - count(*) FILTER (WHERE state = 'FAILED') + count(*) FILTER (WHERE state = 'FAILED' AND updated_at > now() - interval '24 hours') FROM remote_cam.gundi_inbox WHERE received_at > now() - interval '14 days' """, @@ -756,7 +759,7 @@ class Pipeline: SELECT count(*) FILTER (WHERE ocr_state = 'PENDING' AND updated_at < now() - make_interval(secs => %s)), - count(*) FILTER (WHERE ocr_state = 'FAIL') + count(*) FILTER (WHERE ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours') FROM remote_cam.image WHERE source = 'gundi' AND servertime > now() - interval '14 days' """, @@ -771,7 +774,7 @@ class Pipeline: self._last_alarm = now print( f"[GUNDI][ALARM] backlog received_haengt={received_stuck} awaiting_attachment_alt={awaiting_old} " - f"failed={failed} ocr_haengt={ocr_stuck} ocr_fail={ocr_failed} (Schwelle {limit}s, 14 Tage) " - "- bei failed/ocr_fail: tools/gundi_requeue.py", + f"failed_24h={failed} ocr_haengt={ocr_stuck} ocr_fail_24h={ocr_failed} (haengt = laenger als {limit}s) " + "- Details/Zuruecksetzen: tools/gundi_requeue.py", flush=True, ) diff --git a/tests/test_gundi_resilience.py b/tests/test_gundi_resilience.py index a427118..18090e2 100644 --- a/tests/test_gundi_resilience.py +++ b/tests/test_gundi_resilience.py @@ -155,6 +155,45 @@ class FailureRecordingTests(unittest.TestCase): self.assertIn("'FAIL'", sql) +class BacklogAlarmTests(unittest.TestCase): + def run_check(self, inbox_row, image_row, pipeline=None): + cur = FakeCursor(rows=[inbox_row, image_row]) + p = pipeline or bare_pipeline() + p.gcfg = SimpleNamespace(alarm_after_seconds=1800) + p._last_alarm = 0.0 + out = [] + with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(" ".join(map(str, a)))): + p.check_backlog() + return p, cur, out + + def test_silent_when_nothing_is_wrong(self): + _, _, out = self.run_check((0, 0, 0), (0, 0)) + self.assertEqual(out, []) + + def test_alarm_for_stuck_and_new_failures(self): + _, _, out = self.run_check((2, 0, 1), (0, 3)) + self.assertEqual(len(out), 1) + self.assertIn("[GUNDI][ALARM]", out[0]) + self.assertIn("received_haengt=2", out[0]) + self.assertIn("failed_24h=1", out[0]) + self.assertIn("ocr_fail_24h=3", out[0]) + + def test_failures_older_than_24h_are_not_counted(self): + _, cur, _ = self.run_check((0, 0, 0), (0, 0)) + sqls = " ".join(sql for sql, _ in cur.executed) + self.assertIn("state = 'FAILED' AND updated_at > now() - interval '24 hours'", sqls) + self.assertIn("ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours'", sqls) + + def test_alarm_is_throttled(self): + p, _, first = self.run_check((1, 0, 0), (0, 0)) + self.assertEqual(len(first), 1) + cur = FakeCursor(rows=[(1, 0, 0), (0, 0)]) + out = [] + with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(a)): + p.check_backlog() # direkt danach -> unterdrueckt + self.assertEqual(out, []) + + class SubmitDedupeTests(unittest.TestCase): def test_same_id_is_queued_once_until_done(self): p = bare_pipeline()