The poller crashed on every S3 or DB error (1162 restarts during the fsn1 object storage degradation on 2026-09-21). Now: - main loop catches errors per cycle, classifies them (S3 5xx/throttling, connection/timeouts, psycopg.OperationalError = transient; everything else logged with traceback) and retries with exponential backoff (5 s .. 5 min, reset after success); db_init runs until it succeeds once - explicit botocore config for the poller: connect/read timeouts, standard retries (total attempts), s3v4, path addressing; S3(cfg) without a config keeps the previous client for the Gundi path - DB connect timeout + TCP keepalives - one S3 client per cycle shared by all workers - a batch stops after a transient S3/DB error (image stays ERROR as before) - no busy loop when ready sets keep failing - [HEARTBEAT] log line with counters, heartbeat file + Docker healthcheck - tools/entrance_inventory.py: read-only listing of icu/entrance orphans - unit tests with simulated S3/DB failures Per-image transaction, status names/order, keys and paths are unchanged. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
68 lines
2.4 KiB
YAML
68 lines
2.4 KiB
YAML
# config.yaml - Nicht-geheime Konfiguration (App-Tunables + oeffentliche Endpunkte).
|
|
#
|
|
# WICHTIG: Hier gehoeren KEINE Zugangsdaten hinein.
|
|
# Secrets kommen ausschliesslich aus Umgebungsvariablen (.env):
|
|
# S3_ACCESS_KEY, S3_SECRET_KEY, PG_DSN, DEERMAPPER_API_KEY
|
|
# (optional koennen auch S3_ENDPOINT / S3_BUCKET / DEERMAPPER_API_URL per ENV
|
|
# ueberschrieben werden.)
|
|
|
|
s3:
|
|
endpoint: "https://fsn1.your-objectstorage.com/"
|
|
bucket: "trapper-meles"
|
|
# access_key / secret_key -> via ENV (.env): S3_ACCESS_KEY / S3_SECRET_KEY
|
|
|
|
postgres: {}
|
|
# dsn -> via ENV (.env): PG_DSN (inkl. ?sslmode=require)
|
|
|
|
deermapper-api:
|
|
base_url: "https://webapp.deermapper.net/api/icu"
|
|
# apiKey -> via ENV (.env): DEERMAPPER_API_KEY
|
|
|
|
app:
|
|
entrance_prefix: "icu/entrance/"
|
|
processed_prefix: "icu/processed/"
|
|
thumb_prefix: "icu/thumbnails/"
|
|
min_age_seconds: 90
|
|
poll_seconds: 30
|
|
# OCR crop tunables (defaults are usually fine)
|
|
ocr_crop_w_frac: 0.63
|
|
ocr_crop_h_frac: 0.05
|
|
thumb_max: 512
|
|
exiftool_timeout_seconds: 15
|
|
job_sleep_seconds: 0.0
|
|
parallel_workers: 6
|
|
s3_max_pool_connections: 48
|
|
# Optional processing switches:
|
|
# - false => image still gets moved + thumbnail, and backfill flag is set in import_job
|
|
enable_ocr: true
|
|
enable_exif: true
|
|
enable_deermapper_api: false
|
|
deermapper_api_timeout_seconds: 20
|
|
deermapper_api_image_field: "image"
|
|
# Robustheit (nur Poller): S3-Timeouts/Retries, Backoff bei Stoerungen, Lebenszeichen
|
|
s3_connect_timeout_seconds: 10
|
|
s3_read_timeout_seconds: 30
|
|
s3_retry_mode: "standard" # standard | adaptive | legacy
|
|
s3_max_attempts: 4 # inkl. erstem Versuch
|
|
pg_connect_timeout_seconds: 10
|
|
backoff_initial_seconds: 5 # 5 s, 10 s, 20 s ... nach fehlgeschlagenem Zyklus
|
|
backoff_max_seconds: 300 # hoechstens 5 min; Reset nach erfolgreichem Zyklus
|
|
heartbeat_seconds: 300 # [HEARTBEAT]-Logzeile mit Zaehlern
|
|
|
|
# Gundi-v2-Push-Empfaenger (Service gundi-receiver).
|
|
# api key -> via ENV (.env): GUNDI_API_KEY (min. 24 Zeichen)
|
|
gundi:
|
|
enabled: true
|
|
bind_host: "0.0.0.0" # im Container; nach aussen nur 127.0.0.1:8090 (compose)
|
|
bind_port: 8090
|
|
inbox_prefix: "icu/gundi-test/inbox/"
|
|
processed_prefix: "icu/gundi-test/processed/" # spaeter: icu/processed/
|
|
thumb_prefix: "icu/gundi-test/thumbnails/" # spaeter: icu/thumbnails/
|
|
workers: 4
|
|
ocr_workers: 2
|
|
max_request_bytes: 33554432
|
|
abandon_after_hours: 24
|
|
sweep_seconds: 60
|
|
stale_after_seconds: 300
|
|
max_attempts: 3
|