Make the Gundi receiver robust against S3/DB outages

- Upload request path uses its own S3 client with short timeouts (5s/30s,
  2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB
  failures answer 503 + Retry-After instead of 500 after minutes of waiting.
- Transient S3/DB errors in ingest/OCR no longer count against max_attempts
  (previously ~15 min of outage turned images into FAILED); the sweep retries.
- The sweep no longer queues the same id twice (no pile-up during long outages).
- Backlog check logs [GUNDI][ALARM] for stuck/failed entries.
- tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
dom
2026-10-05 16:40:48 +02:00
co-authored by Claude Opus 5.5
parent cb2566a537
commit 6d518b4542
7 changed files with 659 additions and 21 deletions
+15 -7
View File
@@ -29,9 +29,10 @@ from gundi_pipeline import (
utc_now_str,
validate_event,
)
from main import AppConfig, load_config
from main import AppConfig, is_transient_error, load_config
EVENTS_PATH = "/v2/events/"
RETRY_AFTER_SECONDS = 30
ATTACHMENT_RE = re.compile(r"^/v2/events/([^/]+)/attachments(/?)$")
JPEG_MAGIC = b"\xff\xd8\xff"
@@ -73,18 +74,20 @@ class GundiHandler(BaseHTTPRequestHandler):
print(f"[HTTP] {client or self.client_address[0]} {fmt % args}", flush=True)
# --- Antworten -------------------------------------------------------
def _json(self, code: int, data) -> None:
def _json(self, code: int, data, *, retry_after: int | None = None) -> None:
body = json.dumps(data, ensure_ascii=False).encode("utf-8")
self.send_response(code)
self.send_header("Content-Type", "application/json; charset=utf-8")
if retry_after is not None:
self.send_header("Retry-After", str(retry_after))
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _error(self, code: int, message: str, *, close: bool = False) -> None:
def _error(self, code: int, message: str, *, close: bool = False, retry_after: int | None = None) -> None:
if close:
self.close_connection = True
self._json(code, {"error": message})
self._json(code, {"error": message}, retry_after=retry_after)
def _redirect(self, location: str) -> None:
# 308 statt 301: der Client muss POST + Body beibehalten.
@@ -155,8 +158,13 @@ class GundiHandler(BaseHTTPRequestHandler):
else:
self._handle_events(body)
except Exception as e:
print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True)
self._error(500, "internal error", close=True)
if is_transient_error(e):
# S3/DB voruebergehend weg: 503 + Retry-After, damit der Client es spaeter wiederholt
print(f"[GUNDI] Stoerung {path}: {type(e).__name__}: {e}", flush=True)
self._error(503, "temporarily unavailable, please retry", close=True, retry_after=RETRY_AFTER_SECONDS)
else:
print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True)
self._error(500, "internal error", close=True)
# --- Handler ---------------------------------------------------------
def _handle_events(self, body: bytes) -> None:
@@ -202,7 +210,7 @@ class GundiHandler(BaseHTTPRequestHandler):
return
try:
outcome = attach_image(
self.server.cfg.pg_dsn, self.server.pipeline.s3, self.server.gcfg, object_id, jpg
self.server.cfg.pg_dsn, self.server.pipeline.s3_request, self.server.gcfg, object_id, jpg
)
except LookupError:
self._error(404, "Unknown object_id")