From e60cbc3da34e0ce3cd924a506879863a6458f30f Mon Sep 17 00:00:00 2001 From: Bram Kelchtermans Date: Thu, 1 Oct 2026 16:42:30 +0200 Subject: [PATCH] database syncer --- Dockers/database-syncer/.dockerignore | 3 + Dockers/database-syncer/.env.example | 8 ++ Dockers/database-syncer/Dockerfile | 24 +++++ Dockers/database-syncer/README.md | 72 +++++++++++++ Dockers/database-syncer/entrypoint.py | 123 +++++++++++++++++++++++ Dockers/database-syncer/requirements.txt | 1 + Dockers/database-syncer/sync.sh | 59 +++++++++++ Dockers/database-syncer/version | 1 + 8 files changed, 291 insertions(+) create mode 100644 Dockers/database-syncer/.dockerignore create mode 100644 Dockers/database-syncer/.env.example create mode 100644 Dockers/database-syncer/Dockerfile create mode 100644 Dockers/database-syncer/README.md create mode 100755 Dockers/database-syncer/entrypoint.py create mode 100644 Dockers/database-syncer/requirements.txt create mode 100755 Dockers/database-syncer/sync.sh create mode 100644 Dockers/database-syncer/version diff --git a/Dockers/database-syncer/.dockerignore b/Dockers/database-syncer/.dockerignore new file mode 100644 index 0000000..83a921e --- /dev/null +++ b/Dockers/database-syncer/.dockerignore @@ -0,0 +1,3 @@ +.env +*.pyc +__pycache__/ diff --git a/Dockers/database-syncer/.env.example b/Dockers/database-syncer/.env.example new file mode 100644 index 0000000..973f8b7 --- /dev/null +++ b/Dockers/database-syncer/.env.example @@ -0,0 +1,8 @@ +# Required +SOURCE_DB_URL=postgresql://readonly_user:password@source-host:5432/source_db +DEST_DB_URL=postgresql://user:password@dest-host:5432/dest_db +CRON=0 3 * * * + +# Optional +TZ=Europe/Brussels +ON_STARTUP=true diff --git a/Dockers/database-syncer/Dockerfile b/Dockers/database-syncer/Dockerfile new file mode 100644 index 0000000..a6bed11 --- /dev/null +++ b/Dockers/database-syncer/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3.12-slim-bookworm + +RUN apt-get update && apt-get install -y --no-install-recommends \ + postgresql-client \ + ca-certificates \ + tzdata \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN python -m pip install --no-cache-dir -r requirements.txt + +COPY sync.sh /usr/local/bin/sync.sh +COPY entrypoint.py /usr/local/bin/entrypoint.py +RUN chmod +x /usr/local/bin/sync.sh /usr/local/bin/entrypoint.py + +ENV TZ=Europe/Brussels +ENV SOURCE_DB_URL="" +ENV DEST_DB_URL="" +ENV CRON="" +ENV ON_STARTUP=true + +ENTRYPOINT ["/usr/local/bin/entrypoint.py"] diff --git a/Dockers/database-syncer/README.md b/Dockers/database-syncer/README.md new file mode 100644 index 0000000..8fb7426 --- /dev/null +++ b/Dockers/database-syncer/README.md @@ -0,0 +1,72 @@ +# database-syncer + +PostgreSQL sync van **source → destination** op een **CRON**-schema. + +De source-database wordt **nooit** aangepast: alleen `pg_dump` (reads/SELECT), met +`default_transaction_read_only=on` op de source-connectie. + +## Environment variables + +### Required + +| Variable | Description | +|----------|-------------| +| `SOURCE_DB_URL` | PostgreSQL URL van de bron (alleen gelezen) | +| `DEST_DB_URL` | PostgreSQL URL van de bestemming (wordt overschreven) | +| `CRON` | 5-veld cron-expressie (bijv. `0 3 * * *`) of macro (`@hourly`, `@daily`, ...) | + +### Optional + +| Variable | Default | Description | +|----------|---------|-------------| +| `TZ` | `Europe/Brussels` | Tijdzone | +| `ON_STARTUP` | `true` | Direct syncen bij opstart (`true`/`false`) | + +URL-formaat: `postgresql://user:password@host:5432/dbname` + +## Gedrag + +1. Dump van source via `pg_dump` (custom format, read-only). +2. Restore naar destination via `pg_restore --clean --if-exists` (bestaande objecten op dest worden vervangen). +3. Herhaalt volgens `CRON`. + +**Let op:** de destination wordt bij elke sync opgeschoond (DROP + recreate van objecten). Gebruik hiervoor een dedicated shadow/replica-database, niet productie met unieke data. + +## Source read-only garantie + +- Enkel `pg_dump` praat met de source (geen `psql` writes, geen restore). +- `PGOPTIONS=-c default_transaction_read_only=on` op de source-connectie. +- Aanbevolen: geef de source-credentials een **read-only** PostgreSQL-rol. + +## Voorbeeld + +```bash +docker run -d \ + -e SOURCE_DB_URL='postgresql://ro:secret@source:5432/prod' \ + -e DEST_DB_URL='postgresql://rw:secret@dest:5432/shadow' \ + -e 'CRON=0 3 * * *' \ + -e TZ=Europe/Brussels \ + --name database-syncer \ + bramkel/database-syncer:latest +``` + +## Compose + +```yaml + database-syncer: + image: bramkel/database-syncer:latest + container_name: database-syncer + restart: unless-stopped + environment: + - TZ=Europe/Brussels + - SOURCE_DB_URL=postgresql://ro:secret@source:5432/prod + - DEST_DB_URL=postgresql://rw:secret@dest:5432/shadow + - 'CRON=0 3 * * *' + - ON_STARTUP=true +``` + +## Build + +```bash +docker build -t bramkel/database-syncer:latest . +``` diff --git a/Dockers/database-syncer/entrypoint.py b/Dockers/database-syncer/entrypoint.py new file mode 100755 index 0000000..2a40982 --- /dev/null +++ b/Dockers/database-syncer/entrypoint.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Run a PostgreSQL source->dest sync on a cron schedule.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import time +from datetime import datetime +from zoneinfo import ZoneInfo + +from croniter import croniter + +CRON_MACROS = { + "@yearly": "0 0 1 1 *", + "@annually": "0 0 1 1 *", + "@monthly": "0 0 1 * *", + "@weekly": "0 0 * * 0", + "@daily": "0 0 * * *", + "@midnight": "0 0 * * *", + "@hourly": "0 * * * *", +} + +SYNC_SCRIPT = os.environ.get("SYNC_SCRIPT", "/usr/local/bin/sync.sh") + + +def log(msg: str) -> None: + print(f"[{datetime.now().astimezone().isoformat(timespec='seconds')}] {msg}", file=sys.stderr) + + +def zone() -> ZoneInfo: + name = os.environ.get("TZ") or "UTC" + try: + return ZoneInfo(name) + except Exception as exc: + raise SystemExit(f"Invalid TZ '{name}': {exc}") from exc + + +def resolve_cron_expr() -> str: + cron = os.environ.get("CRON", "").strip() + if not cron: + raise SystemExit("CRON must be set (e.g. '0 3 * * *')") + + expr = CRON_MACROS.get(cron.lower(), cron) + if not croniter.is_valid(expr): + raise SystemExit(f"Invalid CRON '{expr}' (expected a 5-field cron expression)") + return expr + + +def require_env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise SystemExit(f"{name} must be set") + return value + + +def run_on_startup() -> bool: + return os.environ.get("ON_STARTUP", "true").lower() in ("1", "true", "yes", "on") + + +def next_run(expr: str, after: datetime) -> datetime: + return croniter(expr, after).get_next(datetime) + + +def redacted(url: str) -> str: + """Hide password in postgresql://user:pass@host/db style URLs.""" + if "://" not in url: + return url + scheme, rest = url.split("://", 1) + if "@" not in rest or ":" not in rest.split("@", 1)[0]: + return url + creds, hostpart = rest.split("@", 1) + user = creds.split(":", 1)[0] + return f"{scheme}://{user}:***@{hostpart}" + + +def do_sync() -> None: + log("Running sync...") + result = subprocess.run([SYNC_SCRIPT], check=False) + if result.returncode != 0: + log(f"Sync failed with exit code {result.returncode}") + else: + log("Sync finished OK") + + +def sleep_until(target: datetime, tz: ZoneInfo) -> None: + while True: + remaining = (target - datetime.now(tz)).total_seconds() + if remaining <= 0: + return + time.sleep(min(remaining, 60.0)) + + +def main() -> None: + source = require_env("SOURCE_DB_URL") + dest = require_env("DEST_DB_URL") + expr = resolve_cron_expr() + tz = zone() + on_startup = run_on_startup() + + log( + "Starting database-syncer " + f"(TZ={tz.key}, CRON='{expr}', " + f"SOURCE='{redacted(source)}', DEST='{redacted(dest)}')" + ) + + if on_startup: + do_sync() + + now = datetime.now(tz) + nxt = next_run(expr, now) + log(f"Next scheduled sync at {nxt.isoformat(timespec='seconds')}") + + while True: + sleep_until(nxt, tz) + do_sync() + nxt = next_run(expr, datetime.now(tz)) + log(f"Next scheduled sync at {nxt.isoformat(timespec='seconds')}") + + +if __name__ == "__main__": + main() diff --git a/Dockers/database-syncer/requirements.txt b/Dockers/database-syncer/requirements.txt new file mode 100644 index 0000000..60fcbaf --- /dev/null +++ b/Dockers/database-syncer/requirements.txt @@ -0,0 +1 @@ +croniter==6.2.4 diff --git a/Dockers/database-syncer/sync.sh b/Dockers/database-syncer/sync.sh new file mode 100755 index 0000000..cbd2fda --- /dev/null +++ b/Dockers/database-syncer/sync.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# Sync PostgreSQL from SOURCE_DB_URL -> DEST_DB_URL. +# Source is read-only: only pg_dump (SELECT) with default_transaction_read_only=on. +set -euo pipefail + +log() { + echo "[$(date -Iseconds)] $*" >&2 +} + +SOURCE_DB_URL="${SOURCE_DB_URL:-}" +DEST_DB_URL="${DEST_DB_URL:-}" + +if [ -z "$SOURCE_DB_URL" ]; then + log "ERROR: SOURCE_DB_URL must be set" + exit 1 +fi +if [ -z "$DEST_DB_URL" ]; then + log "ERROR: DEST_DB_URL must be set" + exit 1 +fi + +DUMP_FILE="${DUMP_FILE:-/tmp/db-sync.dump}" +# Extra safety: force read-only transactions on the source connection. +export PGOPTIONS_SOURCE="${PGOPTIONS_SOURCE:--c default_transaction_read_only=on}" + +log "Starting sync (source -> dest)" +log "Dumping source (read-only)..." + +# pg_dump only reads from source. --clean/--if-exists apply DROP statements to DEST on restore. +PGOPTIONS="$PGOPTIONS_SOURCE" pg_dump \ + --format=custom \ + --no-owner \ + --no-acl \ + --verbose \ + --file="$DUMP_FILE" \ + "$SOURCE_DB_URL" + +log "Restoring into destination (with --clean)..." +pg_restore \ + --clean \ + --if-exists \ + --no-owner \ + --no-acl \ + --verbose \ + --dbname="$DEST_DB_URL" \ + "$DUMP_FILE" || { + # pg_restore returns non-zero on some non-fatal warnings (e.g. missing roles). + # Treat exit code 1 as warning; fail hard on higher codes. + status=$? + if [ "$status" -gt 1 ]; then + log "ERROR: pg_restore failed with exit code $status" + rm -f "$DUMP_FILE" + exit "$status" + fi + log "WARNING: pg_restore finished with warnings (exit $status)" + } + +rm -f "$DUMP_FILE" +log "Sync completed successfully" diff --git a/Dockers/database-syncer/version b/Dockers/database-syncer/version new file mode 100644 index 0000000..3eefcb9 --- /dev/null +++ b/Dockers/database-syncer/version @@ -0,0 +1 @@ +1.0.0