#!/usr/bin/env bash
# backup.sh: consistent dumps, an encrypted local restic snapshot, a verified off-site copy, alerts.
#
# Kit template for guide doc 2 ("Backups and restore"). Checked against the restic, Qdrant, PostgreSQL
# and SQLite docs as of October 2026. Template only: read it, then test it against a scratch repository.
#
#   backup.sh            dumps -> restic backup -> local check -> restic copy off-site -> sampled check
#   backup.sh prune      retention on both repositories, then a full check of each
#
# Configuration: ~/.config/backup/backup.env (chmod 600), written by you. Example:
#   RESTIC_REPOSITORY=/srv/backup/restic               # ideally a second disk, not the system disk
#   RESTIC_PASSWORD_FILE=/home/<user>/.config/backup/restic.password   # keep a copy OFF the server too
#   OFFSITE_REPOSITORY=rclone:drive-offsite:restic/server               # rclone remote, see rclone-drive-setup.md
#   OFFSITE_PASSWORD_FILE=/home/<user>/.config/backup/restic-offsite.password
#   RCLONE_CONFIG=/home/<user>/.config/rclone/rclone.conf
#   BACKUP_PATHS="/home/<user> /etc/ssh/sshd_config.d /etc/cloudflared"
#   EXCLUDE_FILE=/home/<user>/.config/backup/excludes.txt
#   QDRANT_URL=http://127.0.0.1:6333
#   QDRANT_KEY_FILE=/home/<user>/.config/backup/qdrant.key
#   QDRANT_COLLECTIONS="memory handoff"
#   PG_TARGETS="litellm-db:litellm:litellm n8n-postgres-1:n8n:n8n guacamole-db-1:guacamole:guacamole"   # container:user:database
#   SQLITE_FILES="/home/<user>/hub/hub.sqlite"
#   OFFSITE_RCLONE_ARGS="serve restic --stdio --drive-use-trash=false"
#   NOTIFY=/home/<user>/bin/notify.sh
#   NTFY_HEARTBEAT_URL=https://ntfy.sh/<your-random-topic>/backup-heartbeat   # dead-man's switch, see below
#   NTFY_TOKEN=<access token>                          # only for a self-hosted ntfy server
#   HEARTBEAT_URL=https://hc-ping.com/<check-uuid>     # optional: a hosted or self-hosted check service
#
# Dead-man's switch: every successful run publishes a DELAYED ntfy message ("no backup in 26 hours") with
# the same sequence id (the last path segment). Each new run replaces the pending message, so the alert
# reaches your phone only when runs stop succeeding. Needs ntfy server 2.16 or later, and the ntfy server
# must not be the machine it is watching.
#
# Never put the restic password, the rclone token or the heartbeat URL inside the repository they protect.
set -euo pipefail
umask 077

CONF="${BACKUP_CONF:-$HOME/.config/backup/backup.env}"
# shellcheck disable=SC1090
. "$CONF"
export RESTIC_REPOSITORY RESTIC_PASSWORD_FILE RCLONE_CONFIG
OFFSITE=(-r "$OFFSITE_REPOSITORY" --password-file "$OFFSITE_PASSWORD_FILE"
         -o "rclone.args=${OFFSITE_RCLONE_ARGS:-serve restic --stdio --drive-use-trash=false}")
mode="${1:-backup}"
STAGE="$HOME/backup-stage"
STATUS="$HOME/.local/state/backup/status.json"
mkdir -p "$(dirname "$STATUS")"

fail() {
  local msg="$1"
  printf '{"result":"failed","time":"%s","step":"%s"}\n' "$(date -u +%FT%TZ)" "$msg" > "$STATUS"
  [ -n "${NOTIFY:-}" ] && bash "$NOTIFY" "backup failed" "step: $msg" 4 "warning" || true
  [ -n "${HEARTBEAT_URL:-}" ] && curl -fsS -m 10 --retry 3 "$HEARTBEAT_URL/fail" > /dev/null || true
  exit 1
}
trap 'fail "line $LINENO"' ERR

exec 9> "$HOME/.local/state/backup/backup.lock"
flock -n 9 || fail "another backup is running"
[ -n "${HEARTBEAT_URL:-}" ] && curl -fsS -m 10 --retry 3 "$HEARTBEAT_URL/start" > /dev/null || true

if [ "$mode" = "prune" ]; then
  restic forget --keep-daily 7 --keep-weekly 4 --prune
  restic check
  restic "${OFFSITE[@]}" forget --keep-daily 7 --keep-weekly 5 --keep-monthly 12 --prune
  restic "${OFFSITE[@]}" check
  printf '{"result":"ok","mode":"prune","time":"%s"}\n' "$(date -u +%FT%TZ)" > "$STATUS"
  [ -n "${HEARTBEAT_URL:-}" ] && curl -fsS -m 10 --retry 3 "$HEARTBEAT_URL" > /dev/null || true
  exit 0
fi

rm -rf "$STAGE" && mkdir -p "$STAGE"/{qdrant,postgres,sqlite,inventory}

# 1. Qdrant: use its snapshot API, never copy the storage folder of a running server.
if [ -n "${QDRANT_COLLECTIONS:-}" ]; then
  qkey="$(cat "$QDRANT_KEY_FILE")"
  for c in $QDRANT_COLLECTIONS; do
    name="$(curl -fsS -X POST -H "api-key: $qkey" "$QDRANT_URL/collections/$c/snapshots?wait=true" \
      | python3 -c 'import json,sys; print(json.load(sys.stdin)["result"]["name"])')"
    curl -fsS -H "api-key: $qkey" "$QDRANT_URL/collections/$c/snapshots/$name" -o "$STAGE/qdrant/$c.snapshot"
    curl -fsS -X DELETE -H "api-key: $qkey" "$QDRANT_URL/collections/$c/snapshots/$name" > /dev/null
    [ -s "$STAGE/qdrant/$c.snapshot" ] || fail "empty qdrant snapshot $c"
  done
  unset qkey
fi

# 2. PostgreSQL: a custom-format dump per database, read back with pg_restore --list.
for t in ${PG_TARGETS:-}; do
  IFS=: read -r container user db <<< "$t"
  docker exec "$container" pg_dump -U "$user" -d "$db" -Fc > "$STAGE/postgres/$db.dump"
  docker exec -i "$container" pg_restore --list < "$STAGE/postgres/$db.dump" > /dev/null || fail "unreadable dump $db"
done

# 3. SQLite: the online backup command gives a consistent copy of a live database.
for f in ${SQLITE_FILES:-}; do
  out="$STAGE/sqlite/$(basename "$f")"
  sqlite3 "$f" ".backup '$out'"
  [ "$(sqlite3 "$out" 'PRAGMA integrity_check;')" = "ok" ] || fail "sqlite integrity $f"
done

# 4. Inventory: what to reinstall, not the large re-downloadable files themselves.
command -v ollama > /dev/null && ollama list > "$STAGE/inventory/ollama-models.txt" || true
docker ps --format '{{.Names}} {{.Image}}' > "$STAGE/inventory/containers.txt" || true
docker images --digests --format '{{.Repository}}:{{.Tag}} {{.Digest}}' > "$STAGE/inventory/image-digests.txt" || true
systemctl --user list-timers --all --no-pager > "$STAGE/inventory/user-timers.txt" || true
dpkg --get-selections > "$STAGE/inventory/packages.txt" || true

# 5. Encrypted, deduplicated snapshot of files plus the staged dumps.
# shellcheck disable=SC2086
restic backup $BACKUP_PATHS "$STAGE" --exclude-file "$EXCLUDE_FILE" --exclude-caches --tag nightly
restic check

# 6. Off-site copy: restic copy between two repositories. Never "rclone sync" a local repository to the
#    cloud: pruning the local copy would then delete the off-site history too.
restic "${OFFSITE[@]}" copy --from-repo "$RESTIC_REPOSITORY" --from-password-file "$RESTIC_PASSWORD_FILE"
restic "${OFFSITE[@]}" check --read-data-subset=5%

rm -rf "$STAGE"
printf '{"result":"ok","mode":"backup","time":"%s"}\n' "$(date -u +%FT%TZ)" > "$STATUS"
[ -n "${HEARTBEAT_URL:-}" ] && curl -fsS -m 10 --retry 3 "$HEARTBEAT_URL" > /dev/null || true

# 7. Re-arm the dead-man's switch: replace the pending delayed alert with a new one 26 hours out.
if [ -n "${NTFY_HEARTBEAT_URL:-}" ]; then
  hb=(-fsS -m 15 -H "In: 26h" -H "Title: backup missed" -H "Priority: 4" -H "Tags: warning")
  [ -n "${NTFY_TOKEN:-}" ] && hb+=(-H "Authorization: Bearer $NTFY_TOKEN")
  curl "${hb[@]}" -d "No successful backup in the last 26 hours." "$NTFY_HEARTBEAT_URL" > /dev/null || true
fi
exit 0