Make the Gundi receiver robust against S3/DB outages
- Upload request path uses its own S3 client with short timeouts (5s/30s, 2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB failures answer 503 + Retry-After instead of 500 after minutes of waiting. - Transient S3/DB errors in ingest/OCR no longer count against max_attempts (previously ~15 min of outage turned images into FAILED); the sweep retries. - The sweep no longer queues the same id twice (no pile-up during long outages). - Backlog check logs [GUNDI][ALARM] for stuck/failed entries. - tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+15
-7
@@ -29,9 +29,10 @@ from gundi_pipeline import (
|
||||
utc_now_str,
|
||||
validate_event,
|
||||
)
|
||||
from main import AppConfig, load_config
|
||||
from main import AppConfig, is_transient_error, load_config
|
||||
|
||||
EVENTS_PATH = "/v2/events/"
|
||||
RETRY_AFTER_SECONDS = 30
|
||||
ATTACHMENT_RE = re.compile(r"^/v2/events/([^/]+)/attachments(/?)$")
|
||||
JPEG_MAGIC = b"\xff\xd8\xff"
|
||||
|
||||
@@ -73,18 +74,20 @@ class GundiHandler(BaseHTTPRequestHandler):
|
||||
print(f"[HTTP] {client or self.client_address[0]} {fmt % args}", flush=True)
|
||||
|
||||
# --- Antworten -------------------------------------------------------
|
||||
def _json(self, code: int, data) -> None:
|
||||
def _json(self, code: int, data, *, retry_after: int | None = None) -> None:
|
||||
body = json.dumps(data, ensure_ascii=False).encode("utf-8")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||
if retry_after is not None:
|
||||
self.send_header("Retry-After", str(retry_after))
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _error(self, code: int, message: str, *, close: bool = False) -> None:
|
||||
def _error(self, code: int, message: str, *, close: bool = False, retry_after: int | None = None) -> None:
|
||||
if close:
|
||||
self.close_connection = True
|
||||
self._json(code, {"error": message})
|
||||
self._json(code, {"error": message}, retry_after=retry_after)
|
||||
|
||||
def _redirect(self, location: str) -> None:
|
||||
# 308 statt 301: der Client muss POST + Body beibehalten.
|
||||
@@ -155,8 +158,13 @@ class GundiHandler(BaseHTTPRequestHandler):
|
||||
else:
|
||||
self._handle_events(body)
|
||||
except Exception as e:
|
||||
print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True)
|
||||
self._error(500, "internal error", close=True)
|
||||
if is_transient_error(e):
|
||||
# S3/DB voruebergehend weg: 503 + Retry-After, damit der Client es spaeter wiederholt
|
||||
print(f"[GUNDI] Stoerung {path}: {type(e).__name__}: {e}", flush=True)
|
||||
self._error(503, "temporarily unavailable, please retry", close=True, retry_after=RETRY_AFTER_SECONDS)
|
||||
else:
|
||||
print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True)
|
||||
self._error(500, "internal error", close=True)
|
||||
|
||||
# --- Handler ---------------------------------------------------------
|
||||
def _handle_events(self, body: bytes) -> None:
|
||||
@@ -202,7 +210,7 @@ class GundiHandler(BaseHTTPRequestHandler):
|
||||
return
|
||||
try:
|
||||
outcome = attach_image(
|
||||
self.server.cfg.pg_dsn, self.server.pipeline.s3, self.server.gcfg, object_id, jpg
|
||||
self.server.cfg.pg_dsn, self.server.pipeline.s3_request, self.server.gcfg, object_id, jpg
|
||||
)
|
||||
except LookupError:
|
||||
self._error(404, "Unknown object_id")
|
||||
|
||||
Reference in New Issue
Block a user