database syncer

Dockers/database-syncer/.dockerignore

@@ -0,0 +1,3 @@
+.env
+*.pyc
+__pycache__/

Dockers/database-syncer/.env.example

@@ -0,0 +1,8 @@
+# Required
+SOURCE_DB_URL=postgresql://readonly_user:password@source-host:5432/source_db
+DEST_DB_URL=postgresql://user:password@dest-host:5432/dest_db
+CRON=0 3 * * *
+
+# Optional
+TZ=Europe/Brussels
+ON_STARTUP=true

Dockers/database-syncer/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.12-slim-bookworm
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    postgresql-client \
+    ca-certificates \
+    tzdata \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN python -m pip install --no-cache-dir -r requirements.txt
+
+COPY sync.sh /usr/local/bin/sync.sh
+COPY entrypoint.py /usr/local/bin/entrypoint.py
+RUN chmod +x /usr/local/bin/sync.sh /usr/local/bin/entrypoint.py
+
+ENV TZ=Europe/Brussels
+ENV SOURCE_DB_URL=""
+ENV DEST_DB_URL=""
+ENV CRON=""
+ENV ON_STARTUP=true
+
+ENTRYPOINT ["/usr/local/bin/entrypoint.py"]

Dockers/database-syncer/README.md

@@ -0,0 +1,72 @@
+# database-syncer
+
+PostgreSQL sync van **source → destination** op een **CRON**-schema.
+
+De source-database wordt **nooit** aangepast: alleen `pg_dump` (reads/SELECT), met
+`default_transaction_read_only=on` op de source-connectie.
+
+## Environment variables
+
+### Required
+
+| Variable | Description |
+|----------|-------------|
+| `SOURCE_DB_URL` | PostgreSQL URL van de bron (alleen gelezen) |
+| `DEST_DB_URL` | PostgreSQL URL van de bestemming (wordt overschreven) |
+| `CRON` | 5-veld cron-expressie (bijv. `0 3 * * *`) of macro (`@hourly`, `@daily`, ...) |
+
+### Optional
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `TZ` | `Europe/Brussels` | Tijdzone |
+| `ON_STARTUP` | `true` | Direct syncen bij opstart (`true`/`false`) |
+
+URL-formaat: `postgresql://user:password@host:5432/dbname`
+
+## Gedrag
+
+1. Dump van source via `pg_dump` (custom format, read-only).
+2. Restore naar destination via `pg_restore --clean --if-exists` (bestaande objecten op dest worden vervangen).
+3. Herhaalt volgens `CRON`.
+
+**Let op:** de destination wordt bij elke sync opgeschoond (DROP + recreate van objecten). Gebruik hiervoor een dedicated shadow/replica-database, niet productie met unieke data.
+
+## Source read-only garantie
+
+- Enkel `pg_dump` praat met de source (geen `psql` writes, geen restore).
+- `PGOPTIONS=-c default_transaction_read_only=on` op de source-connectie.
+- Aanbevolen: geef de source-credentials een **read-only** PostgreSQL-rol.
+
+## Voorbeeld
+
+```bash
+docker run -d \
+  -e SOURCE_DB_URL='postgresql://ro:secret@source:5432/prod' \
+  -e DEST_DB_URL='postgresql://rw:secret@dest:5432/shadow' \
+  -e 'CRON=0 3 * * *' \
+  -e TZ=Europe/Brussels \
+  --name database-syncer \
+  bramkel/database-syncer:latest
+```
+
+## Compose
+
+```yaml
+  database-syncer:
+    image: bramkel/database-syncer:latest
+    container_name: database-syncer
+    restart: unless-stopped
+    environment:
+      - TZ=Europe/Brussels
+      - SOURCE_DB_URL=postgresql://ro:secret@source:5432/prod
+      - DEST_DB_URL=postgresql://rw:secret@dest:5432/shadow
+      - 'CRON=0 3 * * *'
+      - ON_STARTUP=true
+```
+
+## Build
+
+```bash
+docker build -t bramkel/database-syncer:latest .
+```

Dockers/database-syncer/entrypoint.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+"""Run a PostgreSQL source->dest sync on a cron schedule."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import time
+from datetime import datetime
+from zoneinfo import ZoneInfo
+
+from croniter import croniter
+
+CRON_MACROS = {
+    "@yearly": "0 0 1 1 *",
+    "@annually": "0 0 1 1 *",
+    "@monthly": "0 0 1 * *",
+    "@weekly": "0 0 * * 0",
+    "@daily": "0 0 * * *",
+    "@midnight": "0 0 * * *",
+    "@hourly": "0 * * * *",
+}
+
+SYNC_SCRIPT = os.environ.get("SYNC_SCRIPT", "/usr/local/bin/sync.sh")
+
+
+def log(msg: str) -> None:
+    print(f"[{datetime.now().astimezone().isoformat(timespec='seconds')}] {msg}", file=sys.stderr)
+
+
+def zone() -> ZoneInfo:
+    name = os.environ.get("TZ") or "UTC"
+    try:
+        return ZoneInfo(name)
+    except Exception as exc:
+        raise SystemExit(f"Invalid TZ '{name}': {exc}") from exc
+
+
+def resolve_cron_expr() -> str:
+    cron = os.environ.get("CRON", "").strip()
+    if not cron:
+        raise SystemExit("CRON must be set (e.g. '0 3 * * *')")
+
+    expr = CRON_MACROS.get(cron.lower(), cron)
+    if not croniter.is_valid(expr):
+        raise SystemExit(f"Invalid CRON '{expr}' (expected a 5-field cron expression)")
+    return expr
+
+
+def require_env(name: str) -> str:
+    value = os.environ.get(name, "").strip()
+    if not value:
+        raise SystemExit(f"{name} must be set")
+    return value
+
+
+def run_on_startup() -> bool:
+    return os.environ.get("ON_STARTUP", "true").lower() in ("1", "true", "yes", "on")
+
+
+def next_run(expr: str, after: datetime) -> datetime:
+    return croniter(expr, after).get_next(datetime)
+
+
+def redacted(url: str) -> str:
+    """Hide password in postgresql://user:pass@host/db style URLs."""
+    if "://" not in url:
+        return url
+    scheme, rest = url.split("://", 1)
+    if "@" not in rest or ":" not in rest.split("@", 1)[0]:
+        return url
+    creds, hostpart = rest.split("@", 1)
+    user = creds.split(":", 1)[0]
+    return f"{scheme}://{user}:***@{hostpart}"
+
+
+def do_sync() -> None:
+    log("Running sync...")
+    result = subprocess.run([SYNC_SCRIPT], check=False)
+    if result.returncode != 0:
+        log(f"Sync failed with exit code {result.returncode}")
+    else:
+        log("Sync finished OK")
+
+
+def sleep_until(target: datetime, tz: ZoneInfo) -> None:
+    while True:
+        remaining = (target - datetime.now(tz)).total_seconds()
+        if remaining <= 0:
+            return
+        time.sleep(min(remaining, 60.0))
+
+
+def main() -> None:
+    source = require_env("SOURCE_DB_URL")
+    dest = require_env("DEST_DB_URL")
+    expr = resolve_cron_expr()
+    tz = zone()
+    on_startup = run_on_startup()
+
+    log(
+        "Starting database-syncer "
+        f"(TZ={tz.key}, CRON='{expr}', "
+        f"SOURCE='{redacted(source)}', DEST='{redacted(dest)}')"
+    )
+
+    if on_startup:
+        do_sync()
+
+    now = datetime.now(tz)
+    nxt = next_run(expr, now)
+    log(f"Next scheduled sync at {nxt.isoformat(timespec='seconds')}")
+
+    while True:
+        sleep_until(nxt, tz)
+        do_sync()
+        nxt = next_run(expr, datetime.now(tz))
+        log(f"Next scheduled sync at {nxt.isoformat(timespec='seconds')}")
+
+
+if __name__ == "__main__":
+    main()

Dockers/database-syncer/requirements.txt

@@ -0,0 +1 @@
+croniter==6.2.4

Dockers/database-syncer/sync.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+# Sync PostgreSQL from SOURCE_DB_URL -> DEST_DB_URL.
+# Source is read-only: only pg_dump (SELECT) with default_transaction_read_only=on.
+set -euo pipefail
+
+log() {
+  echo "[$(date -Iseconds)] $*" >&2
+}
+
+SOURCE_DB_URL="${SOURCE_DB_URL:-}"
+DEST_DB_URL="${DEST_DB_URL:-}"
+
+if [ -z "$SOURCE_DB_URL" ]; then
+  log "ERROR: SOURCE_DB_URL must be set"
+  exit 1
+fi
+if [ -z "$DEST_DB_URL" ]; then
+  log "ERROR: DEST_DB_URL must be set"
+  exit 1
+fi
+
+DUMP_FILE="${DUMP_FILE:-/tmp/db-sync.dump}"
+# Extra safety: force read-only transactions on the source connection.
+export PGOPTIONS_SOURCE="${PGOPTIONS_SOURCE:--c default_transaction_read_only=on}"
+
+log "Starting sync (source -> dest)"
+log "Dumping source (read-only)..."
+
+# pg_dump only reads from source. --clean/--if-exists apply DROP statements to DEST on restore.
+PGOPTIONS="$PGOPTIONS_SOURCE" pg_dump \
+  --format=custom \
+  --no-owner \
+  --no-acl \
+  --verbose \
+  --file="$DUMP_FILE" \
+  "$SOURCE_DB_URL"
+
+log "Restoring into destination (with --clean)..."
+pg_restore \
+  --clean \
+  --if-exists \
+  --no-owner \
+  --no-acl \
+  --verbose \
+  --dbname="$DEST_DB_URL" \
+  "$DUMP_FILE" || {
+    # pg_restore returns non-zero on some non-fatal warnings (e.g. missing roles).
+    # Treat exit code 1 as warning; fail hard on higher codes.
+    status=$?
+    if [ "$status" -gt 1 ]; then
+      log "ERROR: pg_restore failed with exit code $status"
+      rm -f "$DUMP_FILE"
+      exit "$status"
+    fi
+    log "WARNING: pg_restore finished with warnings (exit $status)"
+  }
+
+rm -f "$DUMP_FILE"
+log "Sync completed successfully"

Dockers/database-syncer/version

@@ -0,0 +1 @@
+1.0.0