Make the Gundi receiver robust against S3/DB outages
- Upload request path uses its own S3 client with short timeouts (5s/30s, 2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB failures answer 503 + Retry-After instead of 500 after minutes of waiting. - Transient S3/DB errors in ingest/OCR no longer count against max_attempts (previously ~15 min of outage turned images into FAILED); the sweep retries. - The sweep no longer queues the same id twice (no pile-up during long outages). - Backlog check logs [GUNDI][ALARM] for stuck/failed entries. - tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -91,6 +91,24 @@ liest danach asynchron per OCR den Kamera-Zeitstempel, setzt `image.ocr_ts` und
|
|||||||
`DateTimeOriginal`. Ein Sweep holt haengengebliebene Schritte nach und markiert
|
`DateTimeOriginal`. Ein Sweep holt haengengebliebene Schritte nach und markiert
|
||||||
Events ohne Bild nach `abandon_after_hours` als `ABANDONED`.
|
Events ohne Bild nach `abandon_after_hours` als `ABANDONED`.
|
||||||
|
|
||||||
|
### Verhalten bei S3-/DB-Ausfall
|
||||||
|
|
||||||
|
| Teil | Verhalten |
|
||||||
|
|------|-----------|
|
||||||
|
| Status-Events, Event anlegen | brauchen kein S3; bei DB-Ausfall `503` + `Retry-After: 30` |
|
||||||
|
| Bild-Upload (`.../attachments/`) | S3-Zugriff mit kurzen Zeitlimits (`gundi.upload_*`: 5 s Connect, 30 s Read, 2 Versuche), ohne offene DB-Sperre. Bei S3-Stoerung `503` + `Retry-After: 30`; das Event bleibt bis `abandon_after_hours` auf `AWAITING_ATTACHMENT`, der Client kann wiederholen |
|
||||||
|
| Ingest/OCR im Hintergrund | S3-/DB-Stoerungen (`main.is_transient_error`) zaehlen **nicht** gegen `max_attempts`; der Sweep wiederholt alle `stale_after_seconds`. Nur echte Datenfehler fuehren zu `FAILED`/`FAIL` |
|
||||||
|
| Sweep | reiht dieselbe ID nicht doppelt ein (kein Stau bei langem Ausfall) |
|
||||||
|
| Alarm | `[GUNDI][ALARM] backlog ...` im Log (hoechstens alle 5 min), wenn Eintraege laenger als `gundi.alarm_after_seconds` in `RECEIVED`/`AWAITING_ATTACHMENT`/OCR-`PENDING` haengen oder `FAILED`/`FAIL` existieren |
|
||||||
|
|
||||||
|
Gescheiterte Eintraege zuruecksetzen (Trockenlauf ohne `--apply`):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec -w /app melesicumover-gundi python -m tools.gundi_requeue --apply
|
||||||
|
```
|
||||||
|
|
||||||
|
Wichtig: Der Client (ICU) muss nach `5xx`/Timeout den Upload wiederholen; ohne Wiederholung geht das Bild verloren, wenn S3 laenger ausfaellt.
|
||||||
|
|
||||||
### Datenmodell (`schema/001_domain_model.sql`)
|
### Datenmodell (`schema/001_domain_model.sql`)
|
||||||
|
|
||||||
| Tabelle | Inhalt |
|
| Tabelle | Inhalt |
|
||||||
|
|||||||
@@ -65,3 +65,9 @@ gundi:
|
|||||||
sweep_seconds: 60
|
sweep_seconds: 60
|
||||||
stale_after_seconds: 300
|
stale_after_seconds: 300
|
||||||
max_attempts: 3
|
max_attempts: 3
|
||||||
|
# S3-Ausfall-Robustheit (siehe README): Upload-Request scheitert schnell mit 503 + Retry-After;
|
||||||
|
# S3-/DB-Stoerungen zaehlen nicht gegen max_attempts (nur Datenfehler).
|
||||||
|
upload_connect_timeout_seconds: 5
|
||||||
|
upload_read_timeout_seconds: 30
|
||||||
|
upload_max_attempts: 2 # inkl. erstem Versuch
|
||||||
|
alarm_after_seconds: 1800 # [GUNDI][ALARM], wenn Eintraege laenger haengen
|
||||||
|
|||||||
+138
-8
@@ -14,6 +14,7 @@ from typing import Optional
|
|||||||
|
|
||||||
import psycopg
|
import psycopg
|
||||||
import yaml
|
import yaml
|
||||||
|
from botocore.config import Config as BotoClientConfig
|
||||||
from dateutil import tz
|
from dateutil import tz
|
||||||
from psycopg.types.json import Json
|
from psycopg.types.json import Json
|
||||||
|
|
||||||
@@ -21,6 +22,7 @@ from main import (
|
|||||||
AppConfig,
|
AppConfig,
|
||||||
S3,
|
S3,
|
||||||
exif_write_with_exiftool,
|
exif_write_with_exiftool,
|
||||||
|
is_transient_error,
|
||||||
make_thumbnail_bytes,
|
make_thumbnail_bytes,
|
||||||
ocr_extract_timestamp,
|
ocr_extract_timestamp,
|
||||||
to_exif_datetime,
|
to_exif_datetime,
|
||||||
@@ -55,6 +57,12 @@ class GundiConfig:
|
|||||||
sweep_seconds: int = 60
|
sweep_seconds: int = 60
|
||||||
stale_after_seconds: int = 300
|
stale_after_seconds: int = 300
|
||||||
max_attempts: int = 3
|
max_attempts: int = 3
|
||||||
|
# S3 im Request-Pfad (Bild-Upload): schnell scheitern -> 503 + Retry-After statt minutenlang haengen
|
||||||
|
upload_connect_timeout_seconds: int = 5
|
||||||
|
upload_read_timeout_seconds: int = 30
|
||||||
|
upload_max_attempts: int = 2 # inkl. erstem Versuch
|
||||||
|
# Backlog-Alarm: so lange darf ein Eintrag in RECEIVED/AWAITING/OCR-PENDING haengen
|
||||||
|
alarm_after_seconds: int = 1800
|
||||||
|
|
||||||
|
|
||||||
def load_gundi_config(path: str) -> GundiConfig:
|
def load_gundi_config(path: str) -> GundiConfig:
|
||||||
@@ -82,6 +90,10 @@ def load_gundi_config(path: str) -> GundiConfig:
|
|||||||
sweep_seconds=int(g.get("sweep_seconds", defaults.sweep_seconds)),
|
sweep_seconds=int(g.get("sweep_seconds", defaults.sweep_seconds)),
|
||||||
stale_after_seconds=int(g.get("stale_after_seconds", defaults.stale_after_seconds)),
|
stale_after_seconds=int(g.get("stale_after_seconds", defaults.stale_after_seconds)),
|
||||||
max_attempts=max(1, int(g.get("max_attempts", defaults.max_attempts))),
|
max_attempts=max(1, int(g.get("max_attempts", defaults.max_attempts))),
|
||||||
|
upload_connect_timeout_seconds=max(1, int(g.get("upload_connect_timeout_seconds", defaults.upload_connect_timeout_seconds))),
|
||||||
|
upload_read_timeout_seconds=max(1, int(g.get("upload_read_timeout_seconds", defaults.upload_read_timeout_seconds))),
|
||||||
|
upload_max_attempts=max(1, int(g.get("upload_max_attempts", defaults.upload_max_attempts))),
|
||||||
|
alarm_after_seconds=max(60, int(g.get("alarm_after_seconds", defaults.alarm_after_seconds))),
|
||||||
)
|
)
|
||||||
if gcfg.enabled and len(gcfg.api_key) < MIN_API_KEY_LEN:
|
if gcfg.enabled and len(gcfg.api_key) < MIN_API_KEY_LEN:
|
||||||
raise SystemExit(
|
raise SystemExit(
|
||||||
@@ -329,10 +341,15 @@ def store_events(dsn: str, items: list[tuple[dict, dict]]) -> list[dict]:
|
|||||||
|
|
||||||
|
|
||||||
def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes) -> str:
|
def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes) -> str:
|
||||||
"""Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'."""
|
"""Legt das JPEG durable in der Inbox ab. Liefert 'stored' oder 'duplicate'.
|
||||||
|
|
||||||
|
Der S3-Upload laeuft OHNE offene DB-Transaktion/Zeilensperre: bei einer S3-Stoerung
|
||||||
|
blockiert der Request weder eine DB-Verbindung noch den Datensatz. Der Key ist pro
|
||||||
|
object_id fest, ein paralleler/wiederholter Upload ueberschreibt mit gleichem Inhalt.
|
||||||
|
"""
|
||||||
with connect(dsn) as conn, conn.cursor() as cur:
|
with connect(dsn) as conn, conn.cursor() as cur:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s FOR UPDATE",
|
"SELECT event_type, state FROM remote_cam.gundi_inbox WHERE object_id = %s",
|
||||||
(object_id,),
|
(object_id,),
|
||||||
)
|
)
|
||||||
row = cur.fetchone()
|
row = cur.fetchone()
|
||||||
@@ -340,17 +357,22 @@ def attach_image(dsn: str, s3: S3, gcfg: GundiConfig, object_id: str, jpg: bytes
|
|||||||
raise LookupError(object_id)
|
raise LookupError(object_id)
|
||||||
if row[1] in ("RECEIVED", "INGESTED"):
|
if row[1] in ("RECEIVED", "INGESTED"):
|
||||||
return "duplicate"
|
return "duplicate"
|
||||||
|
|
||||||
key = f"{gcfg.inbox_prefix}{object_id}.jpg"
|
key = f"{gcfg.inbox_prefix}{object_id}.jpg"
|
||||||
s3.put_bytes(key, jpg, "image/jpeg")
|
s3.put_bytes(key, jpg, "image/jpeg")
|
||||||
|
|
||||||
|
with connect(dsn) as conn, conn.cursor() as cur:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"""
|
"""
|
||||||
UPDATE remote_cam.gundi_inbox
|
UPDATE remote_cam.gundi_inbox
|
||||||
SET state = 'RECEIVED', attachment_key = %s, attachment_received_at = now(),
|
SET state = 'RECEIVED', attachment_key = %s, attachment_received_at = now(),
|
||||||
attempts = 0, last_error = NULL, updated_at = now()
|
attempts = 0, last_error = NULL, updated_at = now()
|
||||||
WHERE object_id = %s
|
WHERE object_id = %s AND state NOT IN ('RECEIVED', 'INGESTED')
|
||||||
""",
|
""",
|
||||||
(key, object_id),
|
(key, object_id),
|
||||||
)
|
)
|
||||||
|
if cur.rowcount == 0:
|
||||||
|
return "duplicate"
|
||||||
return "stored"
|
return "stored"
|
||||||
|
|
||||||
|
|
||||||
@@ -384,10 +406,33 @@ class Pipeline:
|
|||||||
def __init__(self, cfg: AppConfig, gcfg: GundiConfig):
|
def __init__(self, cfg: AppConfig, gcfg: GundiConfig):
|
||||||
self.cfg = cfg
|
self.cfg = cfg
|
||||||
self.gcfg = gcfg
|
self.gcfg = gcfg
|
||||||
self.s3 = S3(cfg)
|
# Hintergrund (ingest/ocr): geduldig, Poller-Timeouts/-Retries; Request-Pfad (Upload): schnell.
|
||||||
|
# Adressierung/Signatur bleiben wie bisher (nur Timeouts und Retries werden gesetzt).
|
||||||
|
self.s3 = S3(cfg, self._s3_config(
|
||||||
|
cfg, cfg.s3_connect_timeout_seconds, cfg.s3_read_timeout_seconds,
|
||||||
|
cfg.s3_max_attempts, cfg.s3_retry_mode,
|
||||||
|
))
|
||||||
|
self.s3_request = S3(cfg, self._s3_config(
|
||||||
|
cfg, gcfg.upload_connect_timeout_seconds, gcfg.upload_read_timeout_seconds,
|
||||||
|
gcfg.upload_max_attempts, "standard",
|
||||||
|
))
|
||||||
self._ingest_pool = ThreadPoolExecutor(gcfg.workers, thread_name_prefix="ingest")
|
self._ingest_pool = ThreadPoolExecutor(gcfg.workers, thread_name_prefix="ingest")
|
||||||
self._ocr_pool = ThreadPoolExecutor(gcfg.ocr_workers, thread_name_prefix="ocr")
|
self._ocr_pool = ThreadPoolExecutor(gcfg.ocr_workers, thread_name_prefix="ocr")
|
||||||
self._stop = threading.Event()
|
self._stop = threading.Event()
|
||||||
|
# Gleiche IDs nicht mehrfach einreihen (sonst staut der Sweep bei S3-Ausfall alle 60 s neu auf)
|
||||||
|
self._inflight: set[tuple[str, str]] = set()
|
||||||
|
self._inflight_lock = threading.Lock()
|
||||||
|
self._last_alarm = 0.0
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _s3_config(cfg: AppConfig, connect_timeout: int, read_timeout: int,
|
||||||
|
max_attempts: int, retry_mode: str) -> BotoClientConfig:
|
||||||
|
return BotoClientConfig(
|
||||||
|
max_pool_connections=cfg.s3_max_pool_connections,
|
||||||
|
connect_timeout=connect_timeout,
|
||||||
|
read_timeout=read_timeout,
|
||||||
|
retries={"mode": retry_mode, "total_max_attempts": max_attempts},
|
||||||
|
)
|
||||||
|
|
||||||
def start(self) -> None:
|
def start(self) -> None:
|
||||||
threading.Thread(target=self._sweep_loop, name="sweep", daemon=True).start()
|
threading.Thread(target=self._sweep_loop, name="sweep", daemon=True).start()
|
||||||
@@ -398,17 +443,32 @@ class Pipeline:
|
|||||||
self._ocr_pool.shutdown(wait=False, cancel_futures=True)
|
self._ocr_pool.shutdown(wait=False, cancel_futures=True)
|
||||||
|
|
||||||
def submit_ingest(self, object_id: str) -> None:
|
def submit_ingest(self, object_id: str) -> None:
|
||||||
self._ingest_pool.submit(self._guarded, self.ingest, object_id)
|
self._submit(self._ingest_pool, self.ingest, object_id)
|
||||||
|
|
||||||
def submit_ocr(self, image_uuid: str) -> None:
|
def submit_ocr(self, image_uuid: str) -> None:
|
||||||
self._ocr_pool.submit(self._guarded, self.ocr, image_uuid)
|
self._submit(self._ocr_pool, self.ocr, image_uuid)
|
||||||
|
|
||||||
@staticmethod
|
def _submit(self, pool: ThreadPoolExecutor, fn, arg: str) -> None:
|
||||||
def _guarded(fn, arg: str) -> None:
|
token = (fn.__name__, arg)
|
||||||
|
with self._inflight_lock:
|
||||||
|
if token in self._inflight:
|
||||||
|
return
|
||||||
|
self._inflight.add(token)
|
||||||
|
try:
|
||||||
|
pool.submit(self._guarded, token, fn, arg)
|
||||||
|
except Exception:
|
||||||
|
with self._inflight_lock:
|
||||||
|
self._inflight.discard(token)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def _guarded(self, token: tuple[str, str], fn, arg: str) -> None:
|
||||||
try:
|
try:
|
||||||
fn(arg)
|
fn(arg)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[GUNDI] {fn.__name__} {arg} unerwarteter Fehler: {type(e).__name__}: {e}", flush=True)
|
print(f"[GUNDI] {fn.__name__} {arg} unerwarteter Fehler: {type(e).__name__}: {e}", flush=True)
|
||||||
|
finally:
|
||||||
|
with self._inflight_lock:
|
||||||
|
self._inflight.discard(token)
|
||||||
|
|
||||||
# --- Stufe 1 ---------------------------------------------------------
|
# --- Stufe 1 ---------------------------------------------------------
|
||||||
def ingest(self, object_id: str) -> None:
|
def ingest(self, object_id: str) -> None:
|
||||||
@@ -480,6 +540,20 @@ class Pipeline:
|
|||||||
|
|
||||||
def _record_ingest_failure(self, object_id: str, err: Exception) -> None:
|
def _record_ingest_failure(self, object_id: str, err: Exception) -> None:
|
||||||
msg = f"{type(err).__name__}: {err}"[:1000]
|
msg = f"{type(err).__name__}: {err}"[:1000]
|
||||||
|
if is_transient_error(err):
|
||||||
|
# S3/DB gestoert, nicht der Datensatz: Versuch nicht zaehlen (sonst FAILED nach ~15 min
|
||||||
|
# Ausfall). updated_at neu setzen -> der Sweep probiert es nach stale_after_seconds wieder.
|
||||||
|
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
"""
|
||||||
|
UPDATE remote_cam.gundi_inbox
|
||||||
|
SET last_error = %s, updated_at = now()
|
||||||
|
WHERE object_id = %s AND state = 'RECEIVED'
|
||||||
|
""",
|
||||||
|
(msg, object_id),
|
||||||
|
)
|
||||||
|
print(f"[GUNDI] ingest object_id={object_id} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True)
|
||||||
|
return
|
||||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"""
|
"""
|
||||||
@@ -578,6 +652,18 @@ class Pipeline:
|
|||||||
|
|
||||||
def _record_ocr_failure(self, image_uuid: str, err: Exception) -> None:
|
def _record_ocr_failure(self, image_uuid: str, err: Exception) -> None:
|
||||||
msg = f"{type(err).__name__}: {err}"[:1000]
|
msg = f"{type(err).__name__}: {err}"[:1000]
|
||||||
|
if is_transient_error(err):
|
||||||
|
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
"""
|
||||||
|
UPDATE remote_cam.image
|
||||||
|
SET ocr_error = %s, updated_at = now()
|
||||||
|
WHERE image_uuid = %s AND ocr_state = 'PENDING'
|
||||||
|
""",
|
||||||
|
(msg, image_uuid),
|
||||||
|
)
|
||||||
|
print(f"[GUNDI] ocr image_uuid={image_uuid} STOERUNG (S3/DB), kein Versuch gezaehlt, Sweep wiederholt: {msg}", flush=True)
|
||||||
|
return
|
||||||
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||||
cur.execute(
|
cur.execute(
|
||||||
"""
|
"""
|
||||||
@@ -645,3 +731,47 @@ class Pipeline:
|
|||||||
f"[GUNDI] sweep ingest_retry={len(ingest_ids)} ocr_retry={len(ocr_ids)} abandoned={abandoned}",
|
f"[GUNDI] sweep ingest_retry={len(ingest_ids)} ocr_retry={len(ocr_ids)} abandoned={abandoned}",
|
||||||
flush=True,
|
flush=True,
|
||||||
)
|
)
|
||||||
|
self.check_backlog()
|
||||||
|
|
||||||
|
def check_backlog(self) -> None:
|
||||||
|
"""Meldet haengende/gescheiterte Eintraege als [GUNDI][ALARM] (hoechstens alle 5 min)."""
|
||||||
|
limit = self.gcfg.alarm_after_seconds
|
||||||
|
with connect(self.cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||||
|
cur.execute(
|
||||||
|
"""
|
||||||
|
SELECT
|
||||||
|
count(*) FILTER (WHERE state = 'RECEIVED'
|
||||||
|
AND updated_at < now() - make_interval(secs => %s)),
|
||||||
|
count(*) FILTER (WHERE state = 'AWAITING_ATTACHMENT'
|
||||||
|
AND received_at < now() - make_interval(secs => %s)),
|
||||||
|
count(*) FILTER (WHERE state = 'FAILED')
|
||||||
|
FROM remote_cam.gundi_inbox
|
||||||
|
WHERE received_at > now() - interval '14 days'
|
||||||
|
""",
|
||||||
|
(limit, limit),
|
||||||
|
)
|
||||||
|
received_stuck, awaiting_old, failed = cur.fetchone()
|
||||||
|
cur.execute(
|
||||||
|
"""
|
||||||
|
SELECT
|
||||||
|
count(*) FILTER (WHERE ocr_state = 'PENDING'
|
||||||
|
AND updated_at < now() - make_interval(secs => %s)),
|
||||||
|
count(*) FILTER (WHERE ocr_state = 'FAIL')
|
||||||
|
FROM remote_cam.image
|
||||||
|
WHERE source = 'gundi' AND servertime > now() - interval '14 days'
|
||||||
|
""",
|
||||||
|
(limit,),
|
||||||
|
)
|
||||||
|
ocr_stuck, ocr_failed = cur.fetchone()
|
||||||
|
if not (received_stuck or awaiting_old or failed or ocr_stuck or ocr_failed):
|
||||||
|
return
|
||||||
|
now = time.monotonic()
|
||||||
|
if now - self._last_alarm < 300:
|
||||||
|
return
|
||||||
|
self._last_alarm = now
|
||||||
|
print(
|
||||||
|
f"[GUNDI][ALARM] backlog received_haengt={received_stuck} awaiting_attachment_alt={awaiting_old} "
|
||||||
|
f"failed={failed} ocr_haengt={ocr_stuck} ocr_fail={ocr_failed} (Schwelle {limit}s, 14 Tage) "
|
||||||
|
"- bei failed/ocr_fail: tools/gundi_requeue.py",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
|||||||
+13
-5
@@ -29,9 +29,10 @@ from gundi_pipeline import (
|
|||||||
utc_now_str,
|
utc_now_str,
|
||||||
validate_event,
|
validate_event,
|
||||||
)
|
)
|
||||||
from main import AppConfig, load_config
|
from main import AppConfig, is_transient_error, load_config
|
||||||
|
|
||||||
EVENTS_PATH = "/v2/events/"
|
EVENTS_PATH = "/v2/events/"
|
||||||
|
RETRY_AFTER_SECONDS = 30
|
||||||
ATTACHMENT_RE = re.compile(r"^/v2/events/([^/]+)/attachments(/?)$")
|
ATTACHMENT_RE = re.compile(r"^/v2/events/([^/]+)/attachments(/?)$")
|
||||||
JPEG_MAGIC = b"\xff\xd8\xff"
|
JPEG_MAGIC = b"\xff\xd8\xff"
|
||||||
|
|
||||||
@@ -73,18 +74,20 @@ class GundiHandler(BaseHTTPRequestHandler):
|
|||||||
print(f"[HTTP] {client or self.client_address[0]} {fmt % args}", flush=True)
|
print(f"[HTTP] {client or self.client_address[0]} {fmt % args}", flush=True)
|
||||||
|
|
||||||
# --- Antworten -------------------------------------------------------
|
# --- Antworten -------------------------------------------------------
|
||||||
def _json(self, code: int, data) -> None:
|
def _json(self, code: int, data, *, retry_after: int | None = None) -> None:
|
||||||
body = json.dumps(data, ensure_ascii=False).encode("utf-8")
|
body = json.dumps(data, ensure_ascii=False).encode("utf-8")
|
||||||
self.send_response(code)
|
self.send_response(code)
|
||||||
self.send_header("Content-Type", "application/json; charset=utf-8")
|
self.send_header("Content-Type", "application/json; charset=utf-8")
|
||||||
|
if retry_after is not None:
|
||||||
|
self.send_header("Retry-After", str(retry_after))
|
||||||
self.send_header("Content-Length", str(len(body)))
|
self.send_header("Content-Length", str(len(body)))
|
||||||
self.end_headers()
|
self.end_headers()
|
||||||
self.wfile.write(body)
|
self.wfile.write(body)
|
||||||
|
|
||||||
def _error(self, code: int, message: str, *, close: bool = False) -> None:
|
def _error(self, code: int, message: str, *, close: bool = False, retry_after: int | None = None) -> None:
|
||||||
if close:
|
if close:
|
||||||
self.close_connection = True
|
self.close_connection = True
|
||||||
self._json(code, {"error": message})
|
self._json(code, {"error": message}, retry_after=retry_after)
|
||||||
|
|
||||||
def _redirect(self, location: str) -> None:
|
def _redirect(self, location: str) -> None:
|
||||||
# 308 statt 301: der Client muss POST + Body beibehalten.
|
# 308 statt 301: der Client muss POST + Body beibehalten.
|
||||||
@@ -155,6 +158,11 @@ class GundiHandler(BaseHTTPRequestHandler):
|
|||||||
else:
|
else:
|
||||||
self._handle_events(body)
|
self._handle_events(body)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
if is_transient_error(e):
|
||||||
|
# S3/DB voruebergehend weg: 503 + Retry-After, damit der Client es spaeter wiederholt
|
||||||
|
print(f"[GUNDI] Stoerung {path}: {type(e).__name__}: {e}", flush=True)
|
||||||
|
self._error(503, "temporarily unavailable, please retry", close=True, retry_after=RETRY_AFTER_SECONDS)
|
||||||
|
else:
|
||||||
print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True)
|
print(f"[GUNDI] Fehler {path}: {type(e).__name__}: {e}", flush=True)
|
||||||
self._error(500, "internal error", close=True)
|
self._error(500, "internal error", close=True)
|
||||||
|
|
||||||
@@ -202,7 +210,7 @@ class GundiHandler(BaseHTTPRequestHandler):
|
|||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
outcome = attach_image(
|
outcome = attach_image(
|
||||||
self.server.cfg.pg_dsn, self.server.pipeline.s3, self.server.gcfg, object_id, jpg
|
self.server.cfg.pg_dsn, self.server.pipeline.s3_request, self.server.gcfg, object_id, jpg
|
||||||
)
|
)
|
||||||
except LookupError:
|
except LookupError:
|
||||||
self._error(404, "Unknown object_id")
|
self._error(404, "Unknown object_id")
|
||||||
|
|||||||
@@ -0,0 +1,252 @@
|
|||||||
|
"""Receiver bei S3-/DB-Stoerungen: schnelles Scheitern (503), kein Versuchsverbrauch, kein Staudruck.
|
||||||
|
|
||||||
|
python -m unittest discover -s tests -v (oder: python -m pytest tests)
|
||||||
|
|
||||||
|
Braucht die Pakete aus requirements.txt, aber weder Netz noch DB.
|
||||||
|
"""
|
||||||
|
import http.client
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import unittest
|
||||||
|
import uuid
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
import psycopg # noqa: E402
|
||||||
|
from botocore.exceptions import ClientError # noqa: E402
|
||||||
|
|
||||||
|
import gundi_pipeline as gp # noqa: E402
|
||||||
|
import gundi_receiver as gr # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def s3_unavailable() -> ClientError:
|
||||||
|
return ClientError(
|
||||||
|
{"Error": {"Code": "ServiceUnavailable", "Message": "down"}, "ResponseMetadata": {"HTTPStatusCode": 503}},
|
||||||
|
"PutObject",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class FakeCursor:
|
||||||
|
def __init__(self, rows=None, rowcount=1):
|
||||||
|
self.executed: list[tuple[str, tuple]] = []
|
||||||
|
self._rows = list(rows or [])
|
||||||
|
self.rowcount = rowcount
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def execute(self, sql, params=()):
|
||||||
|
self.executed.append((" ".join(sql.split()), params))
|
||||||
|
|
||||||
|
def fetchone(self):
|
||||||
|
return self._rows.pop(0) if self._rows else None
|
||||||
|
|
||||||
|
|
||||||
|
class FakeConn:
|
||||||
|
def __init__(self, cursor: FakeCursor):
|
||||||
|
self._cursor = cursor
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *exc):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def cursor(self):
|
||||||
|
return self._cursor
|
||||||
|
|
||||||
|
|
||||||
|
def fake_connect(cursor: FakeCursor):
|
||||||
|
return lambda dsn: FakeConn(cursor)
|
||||||
|
|
||||||
|
|
||||||
|
GCFG = SimpleNamespace(inbox_prefix="icu/gundi-test/inbox/", max_attempts=3)
|
||||||
|
|
||||||
|
|
||||||
|
class AttachImageTests(unittest.TestCase):
|
||||||
|
def test_s3_down_raises_without_db_lock_and_without_update(self):
|
||||||
|
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")])
|
||||||
|
s3 = mock.Mock()
|
||||||
|
s3.put_bytes.side_effect = s3_unavailable()
|
||||||
|
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||||
|
with self.assertRaises(ClientError):
|
||||||
|
gp.attach_image("dsn", s3, GCFG, "oid", b"\xff\xd8\xff")
|
||||||
|
sqls = [sql for sql, _ in cur.executed]
|
||||||
|
self.assertEqual(len(sqls), 1, sqls) # nur das SELECT, kein UPDATE
|
||||||
|
self.assertNotIn("FOR UPDATE", sqls[0]) # keine Zeilensperre waehrend des S3-Zugriffs
|
||||||
|
self.assertTrue(gr.is_transient_error(ClientError( # und der Fehler gilt als voruebergehend
|
||||||
|
{"Error": {"Code": "ServiceUnavailable"}, "ResponseMetadata": {"HTTPStatusCode": 503}}, "PutObject")))
|
||||||
|
|
||||||
|
def test_success_stores_then_updates(self):
|
||||||
|
cur = FakeCursor(rows=[("cameratrap_rep", "AWAITING_ATTACHMENT")])
|
||||||
|
s3 = mock.Mock()
|
||||||
|
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||||
|
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "stored")
|
||||||
|
s3.put_bytes.assert_called_once_with("icu/gundi-test/inbox/oid.jpg", b"jpg", "image/jpeg")
|
||||||
|
self.assertIn("UPDATE remote_cam.gundi_inbox", cur.executed[-1][0])
|
||||||
|
|
||||||
|
def test_already_received_is_duplicate_without_s3(self):
|
||||||
|
cur = FakeCursor(rows=[("icu_cameratrap_rep", "RECEIVED")])
|
||||||
|
s3 = mock.Mock()
|
||||||
|
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||||
|
self.assertEqual(gp.attach_image("dsn", s3, GCFG, "oid", b"jpg"), "duplicate")
|
||||||
|
s3.put_bytes.assert_not_called()
|
||||||
|
|
||||||
|
def test_parallel_upload_lost_the_race_is_duplicate(self):
|
||||||
|
cur = FakeCursor(rows=[("icu_cameratrap_rep", "AWAITING_ATTACHMENT")], rowcount=0)
|
||||||
|
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||||
|
self.assertEqual(gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg"), "duplicate")
|
||||||
|
|
||||||
|
def test_unknown_or_wrong_type_is_lookup_error(self):
|
||||||
|
for rows in ([], [("camera_status_report", "INGESTED")]):
|
||||||
|
cur = FakeCursor(rows=rows)
|
||||||
|
with mock.patch.object(gp, "connect", fake_connect(cur)):
|
||||||
|
with self.assertRaises(LookupError):
|
||||||
|
gp.attach_image("dsn", mock.Mock(), GCFG, "oid", b"jpg")
|
||||||
|
|
||||||
|
|
||||||
|
def bare_pipeline() -> gp.Pipeline:
|
||||||
|
p = gp.Pipeline.__new__(gp.Pipeline)
|
||||||
|
p.cfg = SimpleNamespace(pg_dsn="dsn")
|
||||||
|
p.gcfg = GCFG
|
||||||
|
p._inflight = set()
|
||||||
|
p._inflight_lock = threading.Lock()
|
||||||
|
return p
|
||||||
|
|
||||||
|
|
||||||
|
class FailureRecordingTests(unittest.TestCase):
|
||||||
|
def run_failure(self, method: str, err: Exception) -> str:
|
||||||
|
cur = FakeCursor(rows=[(1, "RECEIVED")])
|
||||||
|
p = bare_pipeline()
|
||||||
|
with mock.patch.object(gp, "connect", fake_connect(cur)), mock.patch("builtins.print"):
|
||||||
|
getattr(p, method)("id-1", err)
|
||||||
|
self.assertEqual(len(cur.executed), 1)
|
||||||
|
return cur.executed[0][0]
|
||||||
|
|
||||||
|
def test_ingest_transient_does_not_count_attempt(self):
|
||||||
|
sql = self.run_failure("_record_ingest_failure", s3_unavailable())
|
||||||
|
self.assertNotIn("attempts + 1", sql)
|
||||||
|
self.assertNotIn("FAILED", sql)
|
||||||
|
|
||||||
|
def test_ingest_db_outage_does_not_count_attempt(self):
|
||||||
|
sql = self.run_failure("_record_ingest_failure", psycopg.OperationalError("connection timeout expired"))
|
||||||
|
self.assertNotIn("attempts + 1", sql)
|
||||||
|
|
||||||
|
def test_ingest_permanent_counts_attempt(self):
|
||||||
|
sql = self.run_failure("_record_ingest_failure", ValueError("kaputtes Bild"))
|
||||||
|
self.assertIn("attempts + 1", sql)
|
||||||
|
self.assertIn("'FAILED'", sql)
|
||||||
|
|
||||||
|
def test_ocr_transient_does_not_count_attempt(self):
|
||||||
|
sql = self.run_failure("_record_ocr_failure", s3_unavailable())
|
||||||
|
self.assertNotIn("ocr_attempts + 1", sql)
|
||||||
|
self.assertNotIn("'FAIL'", sql)
|
||||||
|
|
||||||
|
def test_ocr_permanent_counts_attempt(self):
|
||||||
|
sql = self.run_failure("_record_ocr_failure", ValueError("kein Zeitstempel"))
|
||||||
|
self.assertIn("ocr_attempts + 1", sql)
|
||||||
|
self.assertIn("'FAIL'", sql)
|
||||||
|
|
||||||
|
|
||||||
|
class SubmitDedupeTests(unittest.TestCase):
|
||||||
|
def test_same_id_is_queued_once_until_done(self):
|
||||||
|
p = bare_pipeline()
|
||||||
|
pool = mock.Mock()
|
||||||
|
calls = []
|
||||||
|
p.ingest = lambda arg: calls.append(arg)
|
||||||
|
p.ingest.__name__ = "ingest"
|
||||||
|
|
||||||
|
p._submit(pool, p.ingest, "a")
|
||||||
|
p._submit(pool, p.ingest, "a") # Sweep reiht dieselbe ID erneut ein -> ignoriert
|
||||||
|
p._submit(pool, p.ingest, "b")
|
||||||
|
self.assertEqual(pool.submit.call_count, 2)
|
||||||
|
|
||||||
|
# Worker laeuft los und ist fertig -> ID darf wieder eingereiht werden
|
||||||
|
token, fn, arg = pool.submit.call_args_list[0].args[1:4]
|
||||||
|
p._guarded(token, fn, arg)
|
||||||
|
self.assertEqual(calls, ["a"])
|
||||||
|
p._submit(pool, p.ingest, "a")
|
||||||
|
self.assertEqual(pool.submit.call_count, 3)
|
||||||
|
|
||||||
|
def test_failing_worker_releases_the_id(self):
|
||||||
|
p = bare_pipeline()
|
||||||
|
pool = mock.Mock()
|
||||||
|
|
||||||
|
def boom(arg):
|
||||||
|
raise RuntimeError("x")
|
||||||
|
|
||||||
|
p._submit(pool, boom, "a")
|
||||||
|
token, fn, arg = pool.submit.call_args.args[1:4]
|
||||||
|
with mock.patch("builtins.print"):
|
||||||
|
p._guarded(token, fn, arg)
|
||||||
|
self.assertEqual(p._inflight, set())
|
||||||
|
|
||||||
|
|
||||||
|
class ReceiverStatusCodeTests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.gcfg = SimpleNamespace(api_key="k" * 32, max_request_bytes=1 << 20)
|
||||||
|
cls.pipeline = SimpleNamespace(s3_request=mock.Mock(), submit_ingest=mock.Mock())
|
||||||
|
cls.server = gr.GundiServer(("127.0.0.1", 0), SimpleNamespace(pg_dsn="dsn"), cls.gcfg, cls.pipeline)
|
||||||
|
cls.port = cls.server.server_address[1]
|
||||||
|
cls.thread = threading.Thread(target=cls.server.serve_forever, daemon=True)
|
||||||
|
cls.thread.start()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
cls.server.shutdown()
|
||||||
|
cls.server.server_close()
|
||||||
|
|
||||||
|
def post(self, path, body, content_type):
|
||||||
|
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||||||
|
try:
|
||||||
|
conn.request("POST", path, body=body, headers={"Content-Type": content_type, "apikey": self.gcfg.api_key})
|
||||||
|
resp = conn.getresponse()
|
||||||
|
return resp.status, resp.getheader("Retry-After"), resp.read()
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
def attachment(self):
|
||||||
|
boundary = "xx"
|
||||||
|
body = (
|
||||||
|
f'--{boundary}\r\nContent-Disposition: form-data; name="file1"; filename="a.jpg"\r\n'
|
||||||
|
"Content-Type: image/jpeg\r\n\r\n"
|
||||||
|
).encode() + b"\xff\xd8\xff\xe0data" + f"\r\n--{boundary}--\r\n".encode()
|
||||||
|
path = f"/v2/events/{uuid.uuid4()}/attachments/"
|
||||||
|
return self.post(path, body, f"multipart/form-data; boundary={boundary}")
|
||||||
|
|
||||||
|
def test_s3_outage_on_upload_gives_503_with_retry_after(self):
|
||||||
|
with mock.patch.object(gr, "attach_image", side_effect=s3_unavailable()), mock.patch("builtins.print"):
|
||||||
|
status, retry_after, _ = self.attachment()
|
||||||
|
self.assertEqual(status, 503)
|
||||||
|
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
|
||||||
|
|
||||||
|
def test_unexpected_error_on_upload_stays_500(self):
|
||||||
|
with mock.patch.object(gr, "attach_image", side_effect=ValueError("bug")), mock.patch("builtins.print"):
|
||||||
|
status, retry_after, _ = self.attachment()
|
||||||
|
self.assertEqual(status, 500)
|
||||||
|
self.assertIsNone(retry_after)
|
||||||
|
|
||||||
|
def test_db_outage_on_event_gives_503(self):
|
||||||
|
event = {"source": "1", "event_type": "camera_status_report", "recorded_at": "2026-10-05T10:00:00Z"}
|
||||||
|
with mock.patch.object(gr, "store_events", side_effect=psycopg.OperationalError("timeout")), \
|
||||||
|
mock.patch("builtins.print"):
|
||||||
|
status, retry_after, body = self.post("/v2/events/", json.dumps(event).encode(), "application/json")
|
||||||
|
self.assertEqual(status, 503)
|
||||||
|
self.assertEqual(retry_after, str(gr.RETRY_AFTER_SECONDS))
|
||||||
|
|
||||||
|
def test_unknown_object_id_is_still_404(self):
|
||||||
|
with mock.patch.object(gr, "attach_image", side_effect=LookupError("x")):
|
||||||
|
status, _, _ = self.attachment()
|
||||||
|
self.assertEqual(status, 404)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,82 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Setzt gescheiterte Gundi-Eintraege (FAILED / OCR FAIL) zurueck, damit der Sweep sie neu verarbeitet.
|
||||||
|
|
||||||
|
python -m tools.gundi_requeue # Trockenlauf: zeigt nur die Zahlen
|
||||||
|
python -m tools.gundi_requeue --apply # setzt zurueck
|
||||||
|
python -m tools.gundi_requeue --apply --since 2026-10-05T12:00:00Z --imei 868711069680778
|
||||||
|
|
||||||
|
Im Container: docker exec -w /app melesicumover-gundi python -m tools.gundi_requeue
|
||||||
|
|
||||||
|
- gundi_inbox FAILED -> RECEIVED (attempts=0). Voraussetzung: die Inbox-Datei liegt noch in S3
|
||||||
|
(sie wird erst nach erfolgreichem Ingest geloescht). Eintraege ohne attachment_key werden uebersprungen.
|
||||||
|
- image ocr_state FAIL -> PENDING (ocr_attempts=0)
|
||||||
|
Der Sweep (alle sweep_seconds) uebernimmt sie nach stale_after_seconds.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||||
|
|
||||||
|
from gundi_pipeline import connect # noqa: E402
|
||||||
|
from main import load_config # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--apply", action="store_true", help="wirklich zuruecksetzen (sonst nur zaehlen)")
|
||||||
|
ap.add_argument("--since", help="nur Eintraege ab diesem Zeitpunkt (ISO 8601, UTC)")
|
||||||
|
ap.add_argument("--imei", help="nur diese IMEI (source)")
|
||||||
|
ap.add_argument("--config", default=os.environ.get("CONFIG_YAML", "/app/config.yaml"))
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
cfg = load_config(args.config)
|
||||||
|
inbox_where = "state = 'FAILED' AND attachment_key IS NOT NULL"
|
||||||
|
image_where = "source = 'gundi' AND ocr_state = 'FAIL'"
|
||||||
|
params_inbox: list = []
|
||||||
|
params_image: list = []
|
||||||
|
if args.since:
|
||||||
|
inbox_where += " AND received_at >= %s"
|
||||||
|
image_where += " AND servertime >= %s"
|
||||||
|
params_inbox.append(args.since)
|
||||||
|
params_image.append(args.since)
|
||||||
|
if args.imei:
|
||||||
|
inbox_where += " AND source = %s"
|
||||||
|
image_where += " AND imei = %s"
|
||||||
|
params_inbox.append(args.imei)
|
||||||
|
params_image.append(args.imei)
|
||||||
|
|
||||||
|
with connect(cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||||
|
cur.execute(f"SELECT count(*) FROM remote_cam.gundi_inbox WHERE {inbox_where}", params_inbox)
|
||||||
|
n_inbox = cur.fetchone()[0]
|
||||||
|
cur.execute(f"SELECT count(*) FROM remote_cam.image WHERE {image_where}", params_image)
|
||||||
|
n_image = cur.fetchone()[0]
|
||||||
|
print(f"gundi_inbox FAILED (mit Inbox-Datei): {n_inbox}")
|
||||||
|
print(f"image OCR FAIL: {n_image}")
|
||||||
|
if not args.apply:
|
||||||
|
print("Trockenlauf - mit --apply zuruecksetzen.")
|
||||||
|
return 0
|
||||||
|
cur.execute(
|
||||||
|
f"""
|
||||||
|
UPDATE remote_cam.gundi_inbox
|
||||||
|
SET state = 'RECEIVED', attempts = 0, last_error = NULL, updated_at = now() - interval '1 hour'
|
||||||
|
WHERE {inbox_where}
|
||||||
|
""",
|
||||||
|
params_inbox,
|
||||||
|
)
|
||||||
|
print(f"zurueckgesetzt: {cur.rowcount} Inbox-Eintraege")
|
||||||
|
cur.execute(
|
||||||
|
f"""
|
||||||
|
UPDATE remote_cam.image
|
||||||
|
SET ocr_state = 'PENDING', ocr_attempts = 0, ocr_error = NULL, updated_at = now() - interval '1 hour'
|
||||||
|
WHERE {image_where}
|
||||||
|
""",
|
||||||
|
params_image,
|
||||||
|
)
|
||||||
|
print(f"zurueckgesetzt: {cur.rowcount} OCR-Eintraege")
|
||||||
|
print("Der Sweep uebernimmt sie in der naechsten Runde.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Live-Ueberwachung des Gundi-Push-Empfaengers: kommen ueberhaupt POSTs von ICU an?
|
||||||
|
#
|
||||||
|
# Verfolgt zwei Quellen parallel und faerbt die relevanten Zeilen ein:
|
||||||
|
# [app] docker logs des Receiver-Containers ([HTTP]-Requestzeilen, [GUNDI]-Events/Anhaenge, Fehler)
|
||||||
|
# [nginx] nginx-Access-Log (meles_dev3), gefiltert auf /v2/events; Fallback sudo, wenn nicht lesbar
|
||||||
|
# [traefik] Access-Log von Traefik (meles2), gefiltert auf /v2/events - zeigt auch Requests, die Traefik
|
||||||
|
# selbst beantwortet und die nie beim Container ankommen (404 = kein Router-Match,
|
||||||
|
# 502/504 = Container down, 308/301 durch http->https-Redirect ...)
|
||||||
|
#
|
||||||
|
# Voraussetzung fuer [traefik]: Access-Log in Traefik aktiviert (--accesslog=true). Liegt er
|
||||||
|
# als Datei im Container (--accesslog.filepath, hier /var/log/access.log), wird per docker exec
|
||||||
|
# mitgelesen; sonst per docker logs.
|
||||||
|
#
|
||||||
|
# Aufruf auf meles2:
|
||||||
|
# tools/watch_gundi.sh # live ab jetzt, inkl. der letzten 10 Minuten
|
||||||
|
# tools/watch_gundi.sh -s 2h # Rueckblick 2 Stunden, dann live
|
||||||
|
# tools/watch_gundi.sh -a # alle Zeilen statt nur POST/Fehler (inkl. /healthz)
|
||||||
|
# tools/watch_gundi.sh -t "" # ohne Traefik-Log
|
||||||
|
# tools/watch_gundi.sh -f /pfad/access.log # Traefik schreibt in eine Datei statt nach stdout
|
||||||
|
#
|
||||||
|
# Optionen:
|
||||||
|
# -n LOGFILE nginx-Access-Log (Default: /var/log/nginx/access.log wenn vorhanden, "" = aus)
|
||||||
|
# -c CONTAINER Receiver-Container (Default: melesicumover-gundi)
|
||||||
|
# -t CONTAINER Traefik-Container (Default: automatisch, erster Container mit "traefik" im Namen)
|
||||||
|
# -f LOGFILE Traefik-Access-Log als Datei statt docker logs
|
||||||
|
# -s SINCE docker-logs --since (Default: 10m)
|
||||||
|
# -a nichts filtern
|
||||||
|
set -u
|
||||||
|
|
||||||
|
CONTAINER="${GUNDI_CONTAINER:-melesicumover-gundi}"
|
||||||
|
NGINX_LOG="${GUNDI_NGINX_LOG-/var/log/nginx/access.log}"
|
||||||
|
TRAEFIK="${GUNDI_TRAEFIK:-auto}"
|
||||||
|
TRAEFIK_FILE=""
|
||||||
|
TRAEFIK_INNER_LOG="${GUNDI_TRAEFIK_LOG:-/var/log/access.log}"
|
||||||
|
SINCE="10m"
|
||||||
|
ALL=0
|
||||||
|
|
||||||
|
while getopts "c:n:t:f:s:ah" opt; do
|
||||||
|
case "$opt" in
|
||||||
|
c) CONTAINER="$OPTARG" ;;
|
||||||
|
n) NGINX_LOG="$OPTARG" ;;
|
||||||
|
t) TRAEFIK="$OPTARG" ;;
|
||||||
|
f) TRAEFIK_FILE="$OPTARG" ;;
|
||||||
|
s) SINCE="$OPTARG" ;;
|
||||||
|
a) ALL=1 ;;
|
||||||
|
*) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
if ! docker inspect "$CONTAINER" >/dev/null 2>&1; then
|
||||||
|
echo "Container '$CONTAINER' nicht gefunden. Laufende Container:" >&2
|
||||||
|
docker ps --format ' {{.Names}}\t{{.Status}}' >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
state=$(docker inspect -f '{{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}n/a{{end}} started={{.State.StartedAt}}' "$CONTAINER")
|
||||||
|
echo "Container $CONTAINER: $state"
|
||||||
|
if curl -fsS -m 3 http://127.0.0.1:8090/healthz >/dev/null 2>&1; then
|
||||||
|
echo "Receiver lokal erreichbar (127.0.0.1:8090/healthz ok)"
|
||||||
|
else
|
||||||
|
echo "WARNUNG: 127.0.0.1:8090/healthz nicht erreichbar"
|
||||||
|
fi
|
||||||
|
echo "Strg+C beendet. Rueckblick: $SINCE"
|
||||||
|
echo "--------------------------------------------------------------------------------"
|
||||||
|
|
||||||
|
# Faerbt und filtert; gibt bei jedem Treffer den Zaehler mit aus.
|
||||||
|
colorize() {
|
||||||
|
awk -v all="$ALL" '
|
||||||
|
BEGIN { R="\033[31m"; G="\033[32m"; Y="\033[33m"; C="\033[36m"; B="\033[1m"; N="\033[0m"; posts=0 }
|
||||||
|
{
|
||||||
|
line = $0
|
||||||
|
is_post = (line ~ /"POST /)
|
||||||
|
is_gundi = (line ~ /\[GUNDI\]/)
|
||||||
|
is_err = (line ~ /(Traceback|Error|Exception|Fehler|WARN|ALARM|STOERUNG)/) || (line ~ /" (4[0-9][0-9]|5[0-9][0-9]) /)
|
||||||
|
if (!all && !is_post && !is_gundi && !is_err) next
|
||||||
|
if (is_post) posts++
|
||||||
|
col = ""
|
||||||
|
if (is_err) col = R
|
||||||
|
else if (is_gundi) col = G
|
||||||
|
else if (is_post) col = C
|
||||||
|
if (line ~ /" 30[18] /) col = Y # Redirect: Client folgt evtl. nicht mit POST
|
||||||
|
printf "%s%s%s", col, line, N
|
||||||
|
if (is_post) printf " %s(POST #%d)%s", B, posts, N
|
||||||
|
printf "\n"
|
||||||
|
fflush()
|
||||||
|
}'
|
||||||
|
}
|
||||||
|
|
||||||
|
pids=()
|
||||||
|
cleanup() { kill "${pids[@]}" 2>/dev/null; exit 0; }
|
||||||
|
trap cleanup INT TERM
|
||||||
|
|
||||||
|
docker logs -f -t --since "$SINCE" "$CONTAINER" 2>&1 \
|
||||||
|
| sed -u 's/^/[app] /' | colorize &
|
||||||
|
pids+=($!)
|
||||||
|
|
||||||
|
if [ -n "$NGINX_LOG" ] && [ -e "$NGINX_LOG" ]; then
|
||||||
|
# Auf meles_dev3 gehoert das Log www-data:adm (0640) - ohne adm-Gruppe per sudo lesen.
|
||||||
|
SUDO=""
|
||||||
|
if [ ! -r "$NGINX_LOG" ]; then
|
||||||
|
# sudo muss im Vordergrund nach dem Passwort fragen (in einer &-Pipeline haengt es);
|
||||||
|
# danach liest "sudo -n" aus dem Credential-Cache.
|
||||||
|
echo "nginx-Log nicht direkt lesbar - sudo-Passwort fuer das Mitlesen:"
|
||||||
|
if sudo -v; then
|
||||||
|
SUDO="sudo -n"
|
||||||
|
else
|
||||||
|
echo "Hinweis: kein sudo - nginx-Log wird nicht verfolgt" >&2
|
||||||
|
NGINX_LOG=""
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [ -n "$NGINX_LOG" ]; then
|
||||||
|
$SUDO tail -n 300 -F "$NGINX_LOG" 2>/dev/null | grep --line-buffered -E '/v2/events' | sed -u 's/^/[nginx] /' | colorize &
|
||||||
|
pids+=($!)
|
||||||
|
fi
|
||||||
|
elif [ -n "$TRAEFIK_FILE" ]; then
|
||||||
|
if [ -r "$TRAEFIK_FILE" ]; then
|
||||||
|
tail -n 200 -F "$TRAEFIK_FILE" 2>/dev/null | grep --line-buffered -E '/v2/events' | sed -u 's/^/[traefik] /' | colorize &
|
||||||
|
pids+=($!)
|
||||||
|
else
|
||||||
|
echo "Hinweis: $TRAEFIK_FILE nicht lesbar - Traefik-Log wird nicht verfolgt" >&2
|
||||||
|
fi
|
||||||
|
elif [ -n "$TRAEFIK" ]; then
|
||||||
|
if [ "$TRAEFIK" = "auto" ]; then
|
||||||
|
TRAEFIK=$(docker ps --format '{{.Names}}' | grep -i traefik | head -n1)
|
||||||
|
fi
|
||||||
|
if [ -n "$TRAEFIK" ] && docker inspect "$TRAEFIK" >/dev/null 2>&1; then
|
||||||
|
echo "Traefik-Container: $TRAEFIK (Access-Log: $TRAEFIK_INNER_LOG)"
|
||||||
|
# Traefik schreibt den Access-Log in eine Datei im Container (--accesslog.filepath),
|
||||||
|
# nicht nach stdout - daher docker exec tail. Fallback: docker logs.
|
||||||
|
if docker exec "$TRAEFIK" test -r "$TRAEFIK_INNER_LOG" 2>/dev/null; then
|
||||||
|
docker exec "$TRAEFIK" tail -n 300 -F "$TRAEFIK_INNER_LOG" 2>/dev/null | grep --line-buffered -E '/v2/events|icu\.meles\.eu' | sed -u 's/^/[traefik] /' | colorize &
|
||||||
|
else
|
||||||
|
docker logs -f -t --since "$SINCE" "$TRAEFIK" 2>&1 | grep --line-buffered -E '/v2/events|icu\.meles\.eu|level=(error|fatal)' | sed -u 's/^/[traefik] /' | colorize &
|
||||||
|
fi
|
||||||
|
pids+=($!)
|
||||||
|
else
|
||||||
|
echo "Hinweis: kein Traefik-Container gefunden (-t NAME oder -f LOGFILE angeben)" >&2
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
wait
|
||||||
Reference in New Issue
Block a user