Backlog alarm: only count new failures (last 24h)
FAILED/OCR FAIL rows never disappear on their own, so counting all of them made the alarm fire every 5 minutes forever. Only stuck entries and failures from the last 24 h trigger it now; older ones stay in the DB for tools/gundi_requeue.py. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -155,6 +155,45 @@ class FailureRecordingTests(unittest.TestCase):
|
||||
self.assertIn("'FAIL'", sql)
|
||||
|
||||
|
||||
class BacklogAlarmTests(unittest.TestCase):
|
||||
def run_check(self, inbox_row, image_row, pipeline=None):
|
||||
cur = FakeCursor(rows=[inbox_row, image_row])
|
||||
p = pipeline or bare_pipeline()
|
||||
p.gcfg = SimpleNamespace(alarm_after_seconds=1800)
|
||||
p._last_alarm = 0.0
|
||||
out = []
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(" ".join(map(str, a)))):
|
||||
p.check_backlog()
|
||||
return p, cur, out
|
||||
|
||||
def test_silent_when_nothing_is_wrong(self):
|
||||
_, _, out = self.run_check((0, 0, 0), (0, 0))
|
||||
self.assertEqual(out, [])
|
||||
|
||||
def test_alarm_for_stuck_and_new_failures(self):
|
||||
_, _, out = self.run_check((2, 0, 1), (0, 3))
|
||||
self.assertEqual(len(out), 1)
|
||||
self.assertIn("[GUNDI][ALARM]", out[0])
|
||||
self.assertIn("received_haengt=2", out[0])
|
||||
self.assertIn("failed_24h=1", out[0])
|
||||
self.assertIn("ocr_fail_24h=3", out[0])
|
||||
|
||||
def test_failures_older_than_24h_are_not_counted(self):
|
||||
_, cur, _ = self.run_check((0, 0, 0), (0, 0))
|
||||
sqls = " ".join(sql for sql, _ in cur.executed)
|
||||
self.assertIn("state = 'FAILED' AND updated_at > now() - interval '24 hours'", sqls)
|
||||
self.assertIn("ocr_state = 'FAIL' AND updated_at > now() - interval '24 hours'", sqls)
|
||||
|
||||
def test_alarm_is_throttled(self):
|
||||
p, _, first = self.run_check((1, 0, 0), (0, 0))
|
||||
self.assertEqual(len(first), 1)
|
||||
cur = FakeCursor(rows=[(1, 0, 0), (0, 0)])
|
||||
out = []
|
||||
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print", side_effect=lambda *a, **k: out.append(a)):
|
||||
p.check_backlog() # direkt danach -> unterdrueckt
|
||||
self.assertEqual(out, [])
|
||||
|
||||
|
||||
class SubmitDedupeTests(unittest.TestCase):
|
||||
def test_same_id_is_queued_once_until_done(self):
|
||||
p = bare_pipeline()
|
||||
|
||||
Reference in New Issue
Block a user