Make the Gundi receiver robust against S3/DB outages
- Upload request path uses its own S3 client with short timeouts (5s/30s, 2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB failures answer 503 + Retry-After instead of 500 after minutes of waiting. - Transient S3/DB errors in ingest/OCR no longer count against max_attempts (previously ~15 min of outage turned images into FAILED); the sweep retries. - The sweep no longer queues the same id twice (no pile-up during long outages). - Backlog check logs [GUNDI][ALARM] for stuck/failed entries. - tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Setzt gescheiterte Gundi-Eintraege (FAILED / OCR FAIL) zurueck, damit der Sweep sie neu verarbeitet.
|
||||
|
||||
python -m tools.gundi_requeue # Trockenlauf: zeigt nur die Zahlen
|
||||
python -m tools.gundi_requeue --apply # setzt zurueck
|
||||
python -m tools.gundi_requeue --apply --since 2026-10-05T12:00:00Z --imei 868711069680778
|
||||
|
||||
Im Container: docker exec -w /app melesicumover-gundi python -m tools.gundi_requeue
|
||||
|
||||
- gundi_inbox FAILED -> RECEIVED (attempts=0). Voraussetzung: die Inbox-Datei liegt noch in S3
|
||||
(sie wird erst nach erfolgreichem Ingest geloescht). Eintraege ohne attachment_key werden uebersprungen.
|
||||
- image ocr_state FAIL -> PENDING (ocr_attempts=0)
|
||||
Der Sweep (alle sweep_seconds) uebernimmt sie nach stale_after_seconds.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from gundi_pipeline import connect # noqa: E402
|
||||
from main import load_config # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true", help="wirklich zuruecksetzen (sonst nur zaehlen)")
|
||||
ap.add_argument("--since", help="nur Eintraege ab diesem Zeitpunkt (ISO 8601, UTC)")
|
||||
ap.add_argument("--imei", help="nur diese IMEI (source)")
|
||||
ap.add_argument("--config", default=os.environ.get("CONFIG_YAML", "/app/config.yaml"))
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = load_config(args.config)
|
||||
inbox_where = "state = 'FAILED' AND attachment_key IS NOT NULL"
|
||||
image_where = "source = 'gundi' AND ocr_state = 'FAIL'"
|
||||
params_inbox: list = []
|
||||
params_image: list = []
|
||||
if args.since:
|
||||
inbox_where += " AND received_at >= %s"
|
||||
image_where += " AND servertime >= %s"
|
||||
params_inbox.append(args.since)
|
||||
params_image.append(args.since)
|
||||
if args.imei:
|
||||
inbox_where += " AND source = %s"
|
||||
image_where += " AND imei = %s"
|
||||
params_inbox.append(args.imei)
|
||||
params_image.append(args.imei)
|
||||
|
||||
with connect(cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(f"SELECT count(*) FROM remote_cam.gundi_inbox WHERE {inbox_where}", params_inbox)
|
||||
n_inbox = cur.fetchone()[0]
|
||||
cur.execute(f"SELECT count(*) FROM remote_cam.image WHERE {image_where}", params_image)
|
||||
n_image = cur.fetchone()[0]
|
||||
print(f"gundi_inbox FAILED (mit Inbox-Datei): {n_inbox}")
|
||||
print(f"image OCR FAIL: {n_image}")
|
||||
if not args.apply:
|
||||
print("Trockenlauf - mit --apply zuruecksetzen.")
|
||||
return 0
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET state = 'RECEIVED', attempts = 0, last_error = NULL, updated_at = now() - interval '1 hour'
|
||||
WHERE {inbox_where}
|
||||
""",
|
||||
params_inbox,
|
||||
)
|
||||
print(f"zurueckgesetzt: {cur.rowcount} Inbox-Eintraege")
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE remote_cam.image
|
||||
SET ocr_state = 'PENDING', ocr_attempts = 0, ocr_error = NULL, updated_at = now() - interval '1 hour'
|
||||
WHERE {image_where}
|
||||
""",
|
||||
params_image,
|
||||
)
|
||||
print(f"zurueckgesetzt: {cur.rowcount} OCR-Eintraege")
|
||||
print("Der Sweep uebernimmt sie in der naechsten Runde.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user