Backlog alarm: only count new failures (last 24h)
FAILED/OCR FAIL rows never disappear on their own, so counting all of them made the alarm fire every 5 minutes forever. Only stuck entries and failures from the last 24 h trigger it now; older ones stay in the DB for tools/gundi_requeue.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -99,7 +99,7 @@ Events ohne Bild nach `abandon_after_hours` als `ABANDONED`.
|
||||
| Bild-Upload (`.../attachments/`) | S3-Zugriff mit kurzen Zeitlimits (`gundi.upload_*`: 5 s Connect, 30 s Read, 2 Versuche), ohne offene DB-Sperre. Bei S3-Stoerung `503` + `Retry-After: 30`; das Event bleibt bis `abandon_after_hours` auf `AWAITING_ATTACHMENT`, der Client kann wiederholen |
|
||||
| Ingest/OCR im Hintergrund | S3-/DB-Stoerungen (`main.is_transient_error`) zaehlen **nicht** gegen `max_attempts`; der Sweep wiederholt alle `stale_after_seconds`. Nur echte Datenfehler fuehren zu `FAILED`/`FAIL` |
|
||||
| Sweep | reiht dieselbe ID nicht doppelt ein (kein Stau bei langem Ausfall) |
|
||||
| Alarm | `[GUNDI][ALARM] backlog ...` im Log (hoechstens alle 5 min), wenn Eintraege laenger als `gundi.alarm_after_seconds` in `RECEIVED`/`AWAITING_ATTACHMENT`/OCR-`PENDING` haengen oder `FAILED`/`FAIL` existieren |
|
||||
| Alarm | `[GUNDI][ALARM] backlog ...` im Log (hoechstens alle 5 min), wenn Eintraege laenger als `gundi.alarm_after_seconds` in `RECEIVED`/`AWAITING_ATTACHMENT`/OCR-`PENDING` haengen oder in den letzten 24 h neue `FAILED`/`FAIL` entstanden sind (aeltere loesen keinen Dauer-Alarm aus) |
|
||||
|
||||
Gescheiterte Eintraege zuruecksetzen (Trockenlauf ohne `--apply`):
|
||||
|
||||
|
||||
+8
-5
@@ -734,7 +734,10 @@ class Pipeline:
|
||||
self.check_backlog()
|
||||
|
||||
def check_backlog(self) -> None:
|
||||
"""Meldet haengende/gescheiterte Eintraege als [GUNDI][ALARM] (hoechstens alle 5 min)."""
|
||||
"""Meldet haengende Eintraege und NEUE Fehlschlaege (letzte 24 h) als [GUNDI][ALARM], hoechstens alle 5 min.
|
||||
|
||||
Aeltere FAILED/FAIL bleiben in der DB (gundi_requeue), loesen aber keinen Dauer-Alarm aus.
|
||||
"""
|
||||
limit = self.gcfg.alarm_after_seconds
|
||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(
|
||||
@@ -744,7 +747,7 @@ class Pipeline:
|
||||
AND updated_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE state = 'AWAITING_ATTACHMENT'
|
||||
AND received_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE state = 'FAILED')
|
||||
count(*) FILTER (WHERE state = 'FAILED' AND updated_at > now() - interval '24 hours')
|
||||
FROM remote_cam.gundi_inbox
|
||||
WHERE received_at > now() - interval '14 days'
|
||||
""",
|
||||
@@ -756,7 +759,7 @@ class Pipeline:
|
||||
SELECT
|
||||
count(*) FILTER (WHERE ocr_state = 'PENDING'
|
||||
AND updated_at < now() - make_interval(secs => %s)),
|
||||
count(*) FILTER (WHERE ocr_state = 'FAIL')
|
||||
count(*) FILTER (WHERE ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours')
|
||||
FROM remote_cam.image
|
||||
WHERE source = 'gundi' AND servertime > now() - interval '14 days'
|
||||
""",
|
||||
@@ -771,7 +774,7 @@ class Pipeline:
|
||||
self._last_alarm = now
|
||||
print(
|
||||
f"[GUNDI][ALARM] backlog received_haengt={received_stuck} awaiting_attachment_alt={awaiting_old} "
|
||||
f"failed={failed} ocr_haengt={ocr_stuck} ocr_fail={ocr_failed} (Schwelle {limit}s, 14 Tage) "
|
||||
"- bei failed/ocr_fail: tools/gundi_requeue.py",
|
||||
f"failed_24h={failed} ocr_haengt={ocr_stuck} ocr_fail_24h={ocr_failed} (haengt = laenger als {limit}s) "
|
||||
"- Details/Zuruecksetzen: tools/gundi_requeue.py",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
@@ -155,6 +155,45 @@ class FailureRecordingTests(unittest.TestCase):
|
||||
self.assertIn("'FAIL'", sql)
|
||||
|
||||
|
||||
class BacklogAlarmTests(unittest.TestCase):
|
||||
def run_check(self, inbox_row, image_row, pipeline=None):
|
||||
cur = FakeCursor(rows=[inbox_row, image_row])
|
||||
p = pipeline or bare_pipeline()
|
||||
p.gcfg = SimpleNamespace(alarm_after_seconds=1800)
|
||||
p._last_alarm = 0.0
|
||||
out = []
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(" ".join(map(str, a)))):
|
||||
p.check_backlog()
|
||||
return p, cur, out
|
||||
|
||||
def test_silent_when_nothing_is_wrong(self):
|
||||
_, _, out = self.run_check((0, 0, 0), (0, 0))
|
||||
self.assertEqual(out, [])
|
||||
|
||||
def test_alarm_for_stuck_and_new_failures(self):
|
||||
_, _, out = self.run_check((2, 0, 1), (0, 3))
|
||||
self.assertEqual(len(out), 1)
|
||||
self.assertIn("[GUNDI][ALARM]", out[0])
|
||||
self.assertIn("received_haengt=2", out[0])
|
||||
self.assertIn("failed_24h=1", out[0])
|
||||
self.assertIn("ocr_fail_24h=3", out[0])
|
||||
|
||||
def test_failures_older_than_24h_are_not_counted(self):
|
||||
_, cur, _ = self.run_check((0, 0, 0), (0, 0))
|
||||
sqls = " ".join(sql for sql, _ in cur.executed)
|
||||
self.assertIn("state = 'FAILED' AND updated_at > now() - interval '24 hours'", sqls)
|
||||
self.assertIn("ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours'", sqls)
|
||||
|
||||
def test_alarm_is_throttled(self):
|
||||
p, _, first = self.run_check((1, 0, 0), (0, 0))
|
||||
self.assertEqual(len(first), 1)
|
||||
cur = FakeCursor(rows=[(1, 0, 0), (0, 0)])
|
||||
out = []
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(a)):
|
||||
p.check_backlog() # direkt danach -> unterdrueckt
|
||||
self.assertEqual(out, [])
|
||||
|
||||
|
||||
class SubmitDedupeTests(unittest.TestCase):
|
||||
def test_same_id_is_queued_once_until_done(self):
|
||||
p = bare_pipeline()
|
||||
|
||||
Reference in New Issue
Block a user