Make the Gundi receiver robust against S3/DB outages
- Upload request path uses its own S3 client with short timeouts (5s/30s, 2 attempts) and no longer holds a DB row lock during the S3 call; S3/DB failures answer 503 + Retry-After instead of 500 after minutes of waiting. - Transient S3/DB errors in ingest/OCR no longer count against max_attempts (previously ~15 min of outage turned images into FAILED); the sweep retries. - The sweep no longer queues the same id twice (no pile-up during long outages). - Backlog check logs [GUNDI][ALARM] for stuck/failed entries. - tools/gundi_requeue.py resets FAILED/FAIL entries; unit tests for all of it. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Setzt gescheiterte Gundi-Eintraege (FAILED / OCR FAIL) zurueck, damit der Sweep sie neu verarbeitet.
|
||||
|
||||
python -m tools.gundi_requeue # Trockenlauf: zeigt nur die Zahlen
|
||||
python -m tools.gundi_requeue --apply # setzt zurueck
|
||||
python -m tools.gundi_requeue --apply --since 2026-10-05T12:00:00Z --imei 868711069680778
|
||||
|
||||
Im Container: docker exec -w /app melesicumover-gundi python -m tools.gundi_requeue
|
||||
|
||||
- gundi_inbox FAILED -> RECEIVED (attempts=0). Voraussetzung: die Inbox-Datei liegt noch in S3
|
||||
(sie wird erst nach erfolgreichem Ingest geloescht). Eintraege ohne attachment_key werden uebersprungen.
|
||||
- image ocr_state FAIL -> PENDING (ocr_attempts=0)
|
||||
Der Sweep (alle sweep_seconds) uebernimmt sie nach stale_after_seconds.
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from gundi_pipeline import connect # noqa: E402
|
||||
from main import load_config # noqa: E402
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--apply", action="store_true", help="wirklich zuruecksetzen (sonst nur zaehlen)")
|
||||
ap.add_argument("--since", help="nur Eintraege ab diesem Zeitpunkt (ISO 8601, UTC)")
|
||||
ap.add_argument("--imei", help="nur diese IMEI (source)")
|
||||
ap.add_argument("--config", default=os.environ.get("CONFIG_YAML", "/app/config.yaml"))
|
||||
args = ap.parse_args()
|
||||
|
||||
cfg = load_config(args.config)
|
||||
inbox_where = "state = 'FAILED' AND attachment_key IS NOT NULL"
|
||||
image_where = "source = 'gundi' AND ocr_state = 'FAIL'"
|
||||
params_inbox: list = []
|
||||
params_image: list = []
|
||||
if args.since:
|
||||
inbox_where += " AND received_at >= %s"
|
||||
image_where += " AND servertime >= %s"
|
||||
params_inbox.append(args.since)
|
||||
params_image.append(args.since)
|
||||
if args.imei:
|
||||
inbox_where += " AND source = %s"
|
||||
image_where += " AND imei = %s"
|
||||
params_inbox.append(args.imei)
|
||||
params_image.append(args.imei)
|
||||
|
||||
with connect(cfg.pg_dsn) as conn, conn.cursor() as cur:
|
||||
cur.execute(f"SELECT count(*) FROM remote_cam.gundi_inbox WHERE {inbox_where}", params_inbox)
|
||||
n_inbox = cur.fetchone()[0]
|
||||
cur.execute(f"SELECT count(*) FROM remote_cam.image WHERE {image_where}", params_image)
|
||||
n_image = cur.fetchone()[0]
|
||||
print(f"gundi_inbox FAILED (mit Inbox-Datei): {n_inbox}")
|
||||
print(f"image OCR FAIL: {n_image}")
|
||||
if not args.apply:
|
||||
print("Trockenlauf - mit --apply zuruecksetzen.")
|
||||
return 0
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE remote_cam.gundi_inbox
|
||||
SET state = 'RECEIVED', attempts = 0, last_error = NULL, updated_at = now() - interval '1 hour'
|
||||
WHERE {inbox_where}
|
||||
""",
|
||||
params_inbox,
|
||||
)
|
||||
print(f"zurueckgesetzt: {cur.rowcount} Inbox-Eintraege")
|
||||
cur.execute(
|
||||
f"""
|
||||
UPDATE remote_cam.image
|
||||
SET ocr_state = 'PENDING', ocr_attempts = 0, ocr_error = NULL, updated_at = now() - interval '1 hour'
|
||||
WHERE {image_where}
|
||||
""",
|
||||
params_image,
|
||||
)
|
||||
print(f"zurueckgesetzt: {cur.rowcount} OCR-Eintraege")
|
||||
print("Der Sweep uebernimmt sie in der naechsten Runde.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env bash
|
||||
# Live-Ueberwachung des Gundi-Push-Empfaengers: kommen ueberhaupt POSTs von ICU an?
|
||||
#
|
||||
# Verfolgt zwei Quellen parallel und faerbt die relevanten Zeilen ein:
|
||||
# [app] docker logs des Receiver-Containers ([HTTP]-Requestzeilen, [GUNDI]-Events/Anhaenge, Fehler)
|
||||
# [nginx] nginx-Access-Log (meles_dev3), gefiltert auf /v2/events; Fallback sudo, wenn nicht lesbar
|
||||
# [traefik] Access-Log von Traefik (meles2), gefiltert auf /v2/events - zeigt auch Requests, die Traefik
|
||||
# selbst beantwortet und die nie beim Container ankommen (404 = kein Router-Match,
|
||||
# 502/504 = Container down, 308/301 durch http->https-Redirect ...)
|
||||
#
|
||||
# Voraussetzung fuer [traefik]: Access-Log in Traefik aktiviert (--accesslog=true). Liegt er
|
||||
# als Datei im Container (--accesslog.filepath, hier /var/log/access.log), wird per docker exec
|
||||
# mitgelesen; sonst per docker logs.
|
||||
#
|
||||
# Aufruf auf meles2:
|
||||
# tools/watch_gundi.sh # live ab jetzt, inkl. der letzten 10 Minuten
|
||||
# tools/watch_gundi.sh -s 2h # Rueckblick 2 Stunden, dann live
|
||||
# tools/watch_gundi.sh -a # alle Zeilen statt nur POST/Fehler (inkl. /healthz)
|
||||
# tools/watch_gundi.sh -t "" # ohne Traefik-Log
|
||||
# tools/watch_gundi.sh -f /pfad/access.log # Traefik schreibt in eine Datei statt nach stdout
|
||||
#
|
||||
# Optionen:
|
||||
# -n LOGFILE nginx-Access-Log (Default: /var/log/nginx/access.log wenn vorhanden, "" = aus)
|
||||
# -c CONTAINER Receiver-Container (Default: melesicumover-gundi)
|
||||
# -t CONTAINER Traefik-Container (Default: automatisch, erster Container mit "traefik" im Namen)
|
||||
# -f LOGFILE Traefik-Access-Log als Datei statt docker logs
|
||||
# -s SINCE docker-logs --since (Default: 10m)
|
||||
# -a nichts filtern
|
||||
set -u
|
||||
|
||||
CONTAINER="${GUNDI_CONTAINER:-melesicumover-gundi}"
|
||||
NGINX_LOG="${GUNDI_NGINX_LOG-/var/log/nginx/access.log}"
|
||||
TRAEFIK="${GUNDI_TRAEFIK:-auto}"
|
||||
TRAEFIK_FILE=""
|
||||
TRAEFIK_INNER_LOG="${GUNDI_TRAEFIK_LOG:-/var/log/access.log}"
|
||||
SINCE="10m"
|
||||
ALL=0
|
||||
|
||||
while getopts "c:n:t:f:s:ah" opt; do
|
||||
case "$opt" in
|
||||
c) CONTAINER="$OPTARG" ;;
|
||||
n) NGINX_LOG="$OPTARG" ;;
|
||||
t) TRAEFIK="$OPTARG" ;;
|
||||
f) TRAEFIK_FILE="$OPTARG" ;;
|
||||
s) SINCE="$OPTARG" ;;
|
||||
a) ALL=1 ;;
|
||||
*) sed -n '2,25p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if ! docker inspect "$CONTAINER" >/dev/null 2>&1; then
|
||||
echo "Container '$CONTAINER' nicht gefunden. Laufende Container:" >&2
|
||||
docker ps --format ' {{.Names}}\t{{.Status}}' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
state=$(docker inspect -f '{{.State.Status}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}n/a{{end}} started={{.State.StartedAt}}' "$CONTAINER")
|
||||
echo "Container $CONTAINER: $state"
|
||||
if curl -fsS -m 3 http://127.0.0.1:8090/healthz >/dev/null 2>&1; then
|
||||
echo "Receiver lokal erreichbar (127.0.0.1:8090/healthz ok)"
|
||||
else
|
||||
echo "WARNUNG: 127.0.0.1:8090/healthz nicht erreichbar"
|
||||
fi
|
||||
echo "Strg+C beendet. Rueckblick: $SINCE"
|
||||
echo "--------------------------------------------------------------------------------"
|
||||
|
||||
# Faerbt und filtert; gibt bei jedem Treffer den Zaehler mit aus.
|
||||
colorize() {
|
||||
awk -v all="$ALL" '
|
||||
BEGIN { R="\033[31m"; G="\033[32m"; Y="\033[33m"; C="\033[36m"; B="\033[1m"; N="\033[0m"; posts=0 }
|
||||
{
|
||||
line = $0
|
||||
is_post = (line ~ /"POST /)
|
||||
is_gundi = (line ~ /\[GUNDI\]/)
|
||||
is_err = (line ~ /(Traceback|Error|Exception|Fehler|WARN|ALARM|STOERUNG)/) || (line ~ /" (4[0-9][0-9]|5[0-9][0-9]) /)
|
||||
if (!all && !is_post && !is_gundi && !is_err) next
|
||||
if (is_post) posts++
|
||||
col = ""
|
||||
if (is_err) col = R
|
||||
else if (is_gundi) col = G
|
||||
else if (is_post) col = C
|
||||
if (line ~ /" 30[18] /) col = Y # Redirect: Client folgt evtl. nicht mit POST
|
||||
printf "%s%s%s", col, line, N
|
||||
if (is_post) printf " %s(POST #%d)%s", B, posts, N
|
||||
printf "\n"
|
||||
fflush()
|
||||
}'
|
||||
}
|
||||
|
||||
pids=()
|
||||
cleanup() { kill "${pids[@]}" 2>/dev/null; exit 0; }
|
||||
trap cleanup INT TERM
|
||||
|
||||
docker logs -f -t --since "$SINCE" "$CONTAINER" 2>&1 \
|
||||
| sed -u 's/^/[app] /' | colorize &
|
||||
pids+=($!)
|
||||
|
||||
if [ -n "$NGINX_LOG" ] && [ -e "$NGINX_LOG" ]; then
|
||||
# Auf meles_dev3 gehoert das Log www-data:adm (0640) - ohne adm-Gruppe per sudo lesen.
|
||||
SUDO=""
|
||||
if [ ! -r "$NGINX_LOG" ]; then
|
||||
# sudo muss im Vordergrund nach dem Passwort fragen (in einer &-Pipeline haengt es);
|
||||
# danach liest "sudo -n" aus dem Credential-Cache.
|
||||
echo "nginx-Log nicht direkt lesbar - sudo-Passwort fuer das Mitlesen:"
|
||||
if sudo -v; then
|
||||
SUDO="sudo -n"
|
||||
else
|
||||
echo "Hinweis: kein sudo - nginx-Log wird nicht verfolgt" >&2
|
||||
NGINX_LOG=""
|
||||
fi
|
||||
fi
|
||||
if [ -n "$NGINX_LOG" ]; then
|
||||
$SUDO tail -n 300 -F "$NGINX_LOG" 2>/dev/null | grep --line-buffered -E '/v2/events' | sed -u 's/^/[nginx] /' | colorize &
|
||||
pids+=($!)
|
||||
fi
|
||||
elif [ -n "$TRAEFIK_FILE" ]; then
|
||||
if [ -r "$TRAEFIK_FILE" ]; then
|
||||
tail -n 200 -F "$TRAEFIK_FILE" 2>/dev/null | grep --line-buffered -E '/v2/events' | sed -u 's/^/[traefik] /' | colorize &
|
||||
pids+=($!)
|
||||
else
|
||||
echo "Hinweis: $TRAEFIK_FILE nicht lesbar - Traefik-Log wird nicht verfolgt" >&2
|
||||
fi
|
||||
elif [ -n "$TRAEFIK" ]; then
|
||||
if [ "$TRAEFIK" = "auto" ]; then
|
||||
TRAEFIK=$(docker ps --format '{{.Names}}' | grep -i traefik | head -n1)
|
||||
fi
|
||||
if [ -n "$TRAEFIK" ] && docker inspect "$TRAEFIK" >/dev/null 2>&1; then
|
||||
echo "Traefik-Container: $TRAEFIK (Access-Log: $TRAEFIK_INNER_LOG)"
|
||||
# Traefik schreibt den Access-Log in eine Datei im Container (--accesslog.filepath),
|
||||
# nicht nach stdout - daher docker exec tail. Fallback: docker logs.
|
||||
if docker exec "$TRAEFIK" test -r "$TRAEFIK_INNER_LOG" 2>/dev/null; then
|
||||
docker exec "$TRAEFIK" tail -n 300 -F "$TRAEFIK_INNER_LOG" 2>/dev/null | grep --line-buffered -E '/v2/events|icu\.meles\.eu' | sed -u 's/^/[traefik] /' | colorize &
|
||||
else
|
||||
docker logs -f -t --since "$SINCE" "$TRAEFIK" 2>&1 | grep --line-buffered -E '/v2/events|icu\.meles\.eu|level=(error|fatal)' | sed -u 's/^/[traefik] /' | colorize &
|
||||
fi
|
||||
pids+=($!)
|
||||
else
|
||||
echo "Hinweis: kein Traefik-Container gefunden (-t NAME oder -f LOGFILE angeben)" >&2
|
||||
fi
|
||||
fi
|
||||
|
||||
wait
|
||||
Reference in New Issue
Block a user