From 5741b3864867dfe37491203720baf76ef0a44de1 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 2 Nov 2026 16:20:59 +0330 Subject: [PATCH 1/3] fix(restore): restore TimescaleDB backups in place instead of a 20 GB side container When the backup's TimescaleDB version differed from the server's, restore pulled timescale/timescaledb-ha:pgNN-tsX.Y-all (about 2.3 GB compressed), loaded a full copy of every database into a temporary container and dumped it again. Users saw a new container and up to 20 GB of extra disk use, and the image was never removed. The stock timescale/timescaledb image already ships the shared libraries of older releases (2.17.0 to 2.31.0 for PostgreSQL 17). When the destination ships the backup's version, restore now creates the extension at that version in the destination database, loads the dump and runs ALTER EXTENSION timescaledb UPDATE in a fresh session. No extra image, container, volume or disk space. The conversion container stays as a fallback for versions the server does not ship. It now checks free disk space before pulling and removes the image afterwards when the restore pulled it. --- .github/workflows/backup-restore.yml | 3 +- docs/backup-and-restore.md | 20 +++- lib/pasarguard-restore.sh | 144 +++++++++++++++++++++++---- tests/backup_restore_roundtrip.sh | 17 ++++ tests/unit_pasarguard.sh | 55 ++++++++++ 5 files changed, 217 insertions(+), 22 deletions(-) diff --git a/.github/workflows/backup-restore.yml b/.github/workflows/backup-restore.yml index b9718f0..5ac97f3 100644 --- a/.github/workflows/backup-restore.yml +++ b/.github/workflows/backup-restore.yml @@ -78,11 +78,12 @@ jobs: sudo apt-get update sudo apt-get install -y rsync zip unzip sqlite3 + # The 2.28.3 image also ships 2.27.2, so this restores in place and + # updates the extension, without a compatibility container. - name: Restore TimescaleDB 2.27 backup into 2.28 destination env: TIMESCALE_SOURCE_IMAGE: timescale/timescaledb:2.27.2-pg17 TIMESCALE_TARGET_IMAGE: timescale/timescaledb:2.28.3-pg17 - TIMESCALEDB_COMPAT_IMAGE: timescale/timescaledb-ha:pg17-ts2.28-all run: bash tests/backup_restore_roundtrip.sh timescaledb single fresh-recovery: diff --git a/docs/backup-and-restore.md b/docs/backup-and-restore.md index 7f2df68..45d908a 100644 --- a/docs/backup-and-restore.md +++ b/docs/backup-and-restore.md @@ -199,8 +199,20 @@ hide an import error. The source extension version is recorded per database in `pg_dump/manifest.tsv`, or in `db_backup.timescaledb-version` for a single-database fallback. For an -ordinary restore to a newer extension, the script attempts conversion in a -temporary compatibility container before changing the destination database. +ordinary restore to a newer extension, the script first checks whether the +destination database container also ships the backup's extension version. The +stock `timescale/timescaledb` images ship many older releases (for PostgreSQL +17, every release from 2.17.0 on). When it does, the database is restored at +the backup's version and the extension is then updated in place with +`ALTER EXTENSION timescaledb UPDATE`. This needs no extra image, container or +disk space. + +Only when the destination does not ship the backup's version does the script +convert the dumps in a temporary compatibility container before changing the +destination database. That path needs several GB of free disk: the +`timescaledb-ha` image (about 2.3 GB to download and 8 GB unpacked), a full +copy of each converted database and the converted dumps. The script checks free +space before pulling anything and removes the image afterwards if it pulled it. That image must contain the source extension version and support its upgrade path. PostgreSQL major-version downgrades are refused; extension conversion is not an arbitrary PostgreSQL downgrade mechanism. @@ -213,8 +225,8 @@ sudo env TIMESCALEDB_BACKUP_VERSION=2.27.2 \ pasarguard restore /root/legacy-backup.zip ``` -For a destination on PostgreSQL 17 / TimescaleDB 2.28, for example, the compatible -image can be supplied explicitly: +When the conversion is needed, the compatibility image can be supplied +explicitly. For a destination on PostgreSQL 17 / TimescaleDB 2.28, for example: ```bash sudo env TIMESCALEDB_COMPAT_IMAGE=timescale/timescaledb-ha:pg17-ts2.28-all \ diff --git a/lib/pasarguard-restore.sh b/lib/pasarguard-restore.sh index 9f8fd24..77bf18b 100644 --- a/lib/pasarguard-restore.sh +++ b/lib/pasarguard-restore.sh @@ -191,6 +191,37 @@ timescaledb_version_is_safe() { [[ "$1" =~ ^[0-9]+[.][0-9]+[.][0-9]+([-.][A-Za-z0-9]+)*$ ]] } +# True (0) when the destination container can create the timescaledb extension +# at exactly this version. The stock image ships the shared libraries of many +# older releases next to the current one. PostgreSQL also lists versions whose +# library is missing, and trying one of those kills the session, so both the +# install script and the library are checked without touching any database. +timescaledb_version_installable() { + local container="$1" admin_user="$2" admin_password="$3" version="$4" log_file="$5" + timescaledb_version_is_safe "$version" || return 1 + local listed="" + listed=$(docker exec -e PGPASSWORD="$admin_password" "$container" \ + psql -X -U "$admin_user" -d postgres -At \ + -c "SELECT count(*) FROM pg_available_extension_versions WHERE name = 'timescaledb' AND version = '$version';" \ + 2>>"$log_file") || return 1 + [ "$listed" = "1" ] || return 1 + # shellcheck disable=SC2016 # $1 expands inside the container shell. + docker exec "$container" sh -c 'test -f "$(pg_config --pkglibdir)/timescaledb-$1.so"' sh "$version" 2>>"$log_file" +} + +# Print the free space in KiB on the filesystem that holds a path. +free_space_kb() { + df -Pk "$1" 2>/dev/null | awk 'NR == 2 {print $4}' +} + +# The compatibility image is several GB once unpacked. Remove it after use when +# this restore pulled it, so a one-off conversion does not keep using the disk. +remove_pulled_timescaledb_compat_image() { + [ -n "$TIMESCALEDB_COMPAT_PULLED_IMAGE" ] || return 0 + docker image rm "$TIMESCALEDB_COMPAT_PULLED_IMAGE" >/dev/null 2>&1 || true + TIMESCALEDB_COMPAT_PULLED_IMAGE="" +} + # Stop and remove the temporary TimescaleDB compatibility container and its # associated Docker volume if they exist. cleanup_timescaledb_compat_container() { @@ -203,6 +234,11 @@ cleanup_timescaledb_compat_container() { TIMESCALEDB_COMPAT_CONTAINER="" TIMESCALEDB_COMPAT_VOLUME="" +TIMESCALEDB_COMPAT_PULLED_IMAGE="" +# Rough disk use of pulling a timescaledb-ha pgNN-*-all image (pg17-ts2.31-all +# is 8 GB unpacked, plus its download), used only for the +# free-space check before it is pulled. +TIMESCALEDB_COMPAT_IMAGE_KB=$((10 * 1048576)) # Signal handler invoked when the process receives HUP, INT, or TERM during # compatibility container operations. Ensures temporary containers and volumes @@ -213,6 +249,7 @@ handle_timescaledb_compat_signal() { cleanup_timescaledb_compat_container "$TIMESCALEDB_COMPAT_CONTAINER" "$TIMESCALEDB_COMPAT_VOLUME" TIMESCALEDB_COMPAT_CONTAINER="" TIMESCALEDB_COMPAT_VOLUME="" + remove_pulled_timescaledb_compat_image if declare -F start_pasarguard_app_services >/dev/null 2>&1; then start_pasarguard_app_services >/dev/null 2>&1 || true fi @@ -220,7 +257,7 @@ handle_timescaledb_compat_signal() { } # Arm signal traps for HUP, INT, and TERM to guarantee cleanup of temporary -# compatibility containers and volumes if interrupted. +# compatibility containers, volumes and pulled images if interrupted. arm_timescaledb_compat_cleanup() { TIMESCALEDB_COMPAT_CONTAINER="$1" TIMESCALEDB_COMPAT_VOLUME="$2" @@ -230,19 +267,22 @@ arm_timescaledb_compat_cleanup() { } # Disarm signal traps and perform final cleanup of the temporary TimescaleDB -# compatibility container and volume. +# compatibility container, its volume and an image this restore pulled. finish_timescaledb_compat_cleanup() { trap - HUP INT TERM cleanup_timescaledb_compat_container "$TIMESCALEDB_COMPAT_CONTAINER" "$TIMESCALEDB_COMPAT_VOLUME" TIMESCALEDB_COMPAT_CONTAINER="" TIMESCALEDB_COMPAT_VOLUME="" + remove_pulled_timescaledb_compat_image } -# Convert cross-version TimescaleDB dumps before the destination is changed. -# The temporary `pgNN-all` image contains historical extension versions: each -# mismatched database is restored at its source version, upgraded to the exact -# target version, and dumped again. The caller may then use the normal restore -# path against a dump whose extension version matches the destination. +# Make cross-version TimescaleDB dumps restorable before the destination is +# changed. When the destination image ships the backup's extension version, no +# conversion is needed: pg_restore_all_user_databases restores at that version +# and updates the extension in place. Only otherwise does a temporary `pgNN-all` +# container restore each mismatched database at its source version, upgrade it +# to the exact target version and dump it again. That path needs several GB of +# disk, so it checks free space first. # # On success PG_PREPARED_DUMP_DIR is set to either source_dir (no conversion was # needed) or output_dir (every mismatched dump converted and validated). @@ -295,14 +335,26 @@ pg_prepare_timescaledb_compatible_dumps() { return 1 fi + local in_place_versions="" while IFS=$'\t' read -r dbname owner has_ts filename source_version; do [ -n "$dbname" ] || continue if [ "$has_ts" = "1" ] && [ -n "$source_version" ] && [ "$source_version" != "$target_version" ]; then + if timescaledb_version_installable "$destination_container" "$admin_user" "$admin_password" \ + "$source_version" "$log_file"; then + in_place_versions+=" $source_version" + continue + fi + echo "Destination does not ship TimescaleDB $source_version; a compatibility conversion is needed" >>"$log_file" needs_conversion=true break fi done <"$manifest" - [ "$needs_conversion" = true ] || return 0 + if [ "$needs_conversion" != true ]; then + if [ -n "$in_place_versions" ]; then + colorized_echo blue "Backup uses TimescaleDB${in_place_versions}; this server has $target_version and ships that version too. Restoring in place, then updating the extension to $target_version." + fi + return 0 + fi local pg_major pg_major=$((server_version_num / 10000)) @@ -330,14 +382,45 @@ pg_prepare_timescaledb_compatible_dumps() { local compat_container="pasarguard-ts-compat-${compat_suffix}" local compat_volume="" - colorized_echo blue "Preparing TimescaleDB $target_version-compatible dumps before changing the destination..." - colorized_echo blue "Pulling temporary compatibility image: $compat_image" - if ! docker pull "$compat_image" >>"$log_file" 2>&1; then - echo "Could not pull TimescaleDB compatibility image: $compat_image" >>"$log_file" + # The temporary container holds a full copy of each converted database, the + # converted dumps land next to the originals, and the image is several GB + # once unpacked. Running out of disk here would also hurt the live database + # on the same disk, so refuse early with numbers the operator can act on. + local image_present=false + docker image inspect "$compat_image" >/dev/null 2>&1 && image_present=true + local dump_kb=0 docker_root="" docker_free_kb="" staging_free_kb="" + dump_kb=$(du -sk "$source_dir" 2>/dev/null | awk '{print $1}') + dump_kb="${dump_kb:-0}" + local docker_need_kb=$((dump_kb * 2)) + [ "$image_present" = true ] || docker_need_kb=$((docker_need_kb + TIMESCALEDB_COMPAT_IMAGE_KB)) + docker_root=$(docker info -f '{{.DockerRootDir}}' 2>>"$log_file") || docker_root="" + [ -n "$docker_root" ] && docker_free_kb=$(free_space_kb "$docker_root") + staging_free_kb=$(free_space_kb "$(dirname "$output_dir")") + if [[ "$docker_free_kb" =~ ^[0-9]+$ ]] && [ "$docker_free_kb" -lt "$docker_need_kb" ]; then + colorized_echo red "Not enough disk space for the TimescaleDB compatibility conversion: it needs about $((docker_need_kb / 1048576 + 1)) GB free under $docker_root, and $((docker_free_kb / 1048576)) GB is free." + echo "Compatibility conversion needs ${docker_need_kb} KiB under $docker_root; ${docker_free_kb} KiB free" >>"$log_file" return 1 fi + if [[ "$staging_free_kb" =~ ^[0-9]+$ ]] && [ "$staging_free_kb" -lt "$dump_kb" ]; then + colorized_echo red "Not enough disk space for the converted TimescaleDB dumps: they need about $((dump_kb / 1048576 + 1)) GB free next to the restore files." + echo "Converted dumps need ${dump_kb} KiB in $(dirname "$output_dir"); ${staging_free_kb} KiB free" >>"$log_file" + return 1 + fi + + colorized_echo blue "Preparing TimescaleDB $target_version-compatible dumps before changing the destination..." + colorized_echo yellow "This server does not ship the backup's TimescaleDB version, so a temporary conversion container is used. It is removed when the restore finishes." + if [ "$image_present" = false ]; then + colorized_echo blue "Pulling temporary compatibility image: $compat_image" + TIMESCALEDB_COMPAT_PULLED_IMAGE="$compat_image" + if ! docker pull "$compat_image" >>"$log_file" 2>&1; then + echo "Could not pull TimescaleDB compatibility image: $compat_image" >>"$log_file" + remove_pulled_timescaledb_compat_image + return 1 + fi + fi if ! compat_volume=$(docker volume create --label "com.pasarguard.restore=$compat_suffix" 2>>"$log_file"); then echo "Could not create temporary TimescaleDB compatibility volume" >>"$log_file" + remove_pulled_timescaledb_compat_image return 1 fi if ! docker run -d --name "$compat_container" --restart=no \ @@ -350,7 +433,8 @@ pg_prepare_timescaledb_compatible_dumps() { -v "$compat_volume:$compat_pgdata" \ "$compat_image" >>"$log_file" 2>&1; then echo "Could not start temporary TimescaleDB compatibility container" >>"$log_file" - cleanup_timescaledb_compat_container "" "$compat_volume" + cleanup_timescaledb_compat_container "$compat_container" "$compat_volume" + remove_pulled_timescaledb_compat_image return 1 fi arm_timescaledb_compat_cleanup "$compat_container" "$compat_volume" @@ -603,9 +687,12 @@ pg_restore_all_user_databases() { # TimescaleDB cross-version safety gate. If this backup recorded a # timescaledb version, refuse to touch the database unless THIS server's - # bundled version matches. Runs BEFORE any terminate/DROP so a mismatch - # never wipes or half-restores data. The preflight rejects versionless - # Timescale manifests before this function is called. + # bundled version matches or the server also ships the backup's version + # (restored at that version, then updated in place). Runs BEFORE any + # terminate/DROP so a mismatch never wipes or half-restores data. The + # preflight rejects versionless Timescale manifests before this + # function is called. + local update_extension_to="" if [ "$has_ts" = "1" ] && [ -n "$ts_version" ]; then # Read THIS server's bundled timescaledb version (read-only). Separate # a probe that FAILED (transient docker/psql error -> non-zero exit) @@ -622,7 +709,11 @@ pg_restore_all_user_databases() { else target_ts="" fi - if [ "$probe_ok" = "1" ] && ! timescaledb_version_matches "$ts_version" "$target_ts"; then + if [ "$probe_ok" = "1" ] && ! timescaledb_version_matches "$ts_version" "$target_ts" && + timescaledb_version_is_safe "$target_ts" && + timescaledb_version_installable "$container_name" "$admin_user" "$admin_password" "$ts_version" "$log_file"; then + update_extension_to="$target_ts" + elif [ "$probe_ok" = "1" ] && ! timescaledb_version_matches "$ts_version" "$target_ts"; then local svn="" pg_major="" svn=$(docker exec -e PGPASSWORD="$admin_password" "$container_name" \ psql -X -U "$admin_user" -d postgres -At -c "SHOW server_version_num;" \ @@ -657,6 +748,15 @@ pg_restore_all_user_databases() { if [ -n "$ts_version" ]; then create_extension_sql="CREATE EXTENSION IF NOT EXISTS timescaledb VERSION '$ts_version';" fi + # The new database is copied from template1, which the stock image + # creates with the current extension. Drop it in its own session so + # the older library loads cleanly in the next one. + if [ -n "$update_extension_to" ] && ! docker exec -e PGPASSWORD="$admin_password" "$container_name" \ + psql -X -v ON_ERROR_STOP=1 -U "$admin_user" --dbname="$destination_dbname" \ + -c "DROP EXTENSION IF EXISTS timescaledb CASCADE;" >>"$log_file" 2>&1; then + colorized_echo red "Could not prepare TimescaleDB database '$destination_dbname'." + continue + fi if ! docker exec -e PGPASSWORD="$admin_password" "$container_name" psql -X -v ON_ERROR_STOP=1 -U "$admin_user" --dbname="$destination_dbname" \ -c "$create_extension_sql" >>"$log_file" 2>&1 || ! docker exec -e PGPASSWORD="$admin_password" "$container_name" psql -X -v ON_ERROR_STOP=1 -U "$admin_user" --dbname="$destination_dbname" \ @@ -676,6 +776,16 @@ pg_restore_all_user_databases() { rm -f "$filtered" docker exec -e PGPASSWORD="$admin_password" "$container_name" psql -X -U "$admin_user" --dbname="$destination_dbname" \ -c "SELECT timescaledb_post_restore();" >>"$log_file" 2>&1 + # ALTER EXTENSION must be the first command of a fresh session. + if [ "$restored" = true ] && [ -n "$update_extension_to" ]; then + colorized_echo blue "Updating TimescaleDB in '$destination_dbname' from $ts_version to $update_extension_to..." + if ! docker exec -e PGPASSWORD="$admin_password" "$container_name" \ + psql -X -v ON_ERROR_STOP=1 -U "$admin_user" --dbname="$destination_dbname" \ + -c "ALTER EXTENSION timescaledb UPDATE TO '$update_extension_to';" >>"$log_file" 2>&1; then + echo "ALTER EXTENSION timescaledb UPDATE TO '$update_extension_to' failed for '$destination_dbname'" >>"$log_file" + restored=false + fi + fi else if docker exec -i -e PGPASSWORD="$restore_password" "$container_name" \ psql -X -v ON_ERROR_STOP=1 -U "$restore_user" --dbname="$destination_dbname" < "$dump_path" >>"$log_file" 2>&1; then diff --git a/tests/backup_restore_roundtrip.sh b/tests/backup_restore_roundtrip.sh index 234c9bd..363c1cb 100644 --- a/tests/backup_restore_roundtrip.sh +++ b/tests/backup_restore_roundtrip.sh @@ -477,6 +477,12 @@ setup_postgresql_container() { wait_for_command 30 docker exec -e PGPASSWORD="$password" "$CONTAINER_NAME" \ psql -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" \ -c "CREATE TABLE ci_roundtrip (id INT PRIMARY KEY, value TEXT NOT NULL); INSERT INTO ci_roundtrip (id, value) VALUES (1, '$initial_value');" + + if [ "$DB_TYPE" = "timescaledb" ]; then + docker exec -e PGPASSWORD="$password" "$CONTAINER_NAME" \ + psql -v ON_ERROR_STOP=1 -U "$DB_USER" -d "$DB_NAME" \ + -c "CREATE TABLE ci_metrics (time TIMESTAMPTZ NOT NULL, value INT NOT NULL); SELECT create_hypertable('ci_metrics', 'time'); INSERT INTO ci_metrics VALUES ('2026-01-01', 1), ('2026-02-01', 2);" >/dev/null + fi } # Query test row value from PostgreSQL or TimescaleDB container. @@ -805,6 +811,17 @@ verify_restored_database() { assert_equals "$restored_value" "$EXPECTED_DB_VALUE" "Database value was not restored from backup." + if [ "$DB_TYPE" = "timescaledb" ]; then + # The extension ends at the destination's version, the hypertable and + # its rows survive, and no temporary conversion container is left. + local ts_state="" + ts_state=$(docker exec -e PGPASSWORD="$CURRENT_DB_PASSWORD" "$CONTAINER_NAME" \ + psql -h 127.0.0.1 -At -U "$DB_USER" -d "$DB_NAME" \ + -c "SELECT (SELECT extversion FROM pg_extension WHERE extname = 'timescaledb') = (SELECT default_version FROM pg_available_extensions WHERE name = 'timescaledb'), (SELECT count(*) FROM timescaledb_information.hypertables WHERE hypertable_name = 'ci_metrics'), (SELECT sum(value) FROM ci_metrics);") + assert_equals "$ts_state" "t|1|3" "TimescaleDB extension version, hypertable or rows not restored." + assert_equals "$(docker ps -a --filter name=pasarguard-ts-compat- --format '{{.Names}}')" "" "TimescaleDB conversion container left behind." + fi + case "$DB_TYPE" in mysql) docker exec -e MYSQL_PWD="$CURRENT_DB_PASSWORD" "$CONTAINER_NAME" mysql -h 127.0.0.1 -u "$DB_USER" "$DB_NAME" -e "SELECT 1;" >/dev/null diff --git a/tests/unit_pasarguard.sh b/tests/unit_pasarguard.sh index 67be95f..f5077ae 100644 --- a/tests/unit_pasarguard.sh +++ b/tests/unit_pasarguard.sh @@ -694,6 +694,61 @@ assert_false "ts_prepare: versionless Timescale manifest fails closed" \ "$WORK_DIR/ts-prepare/output/pg_dump" "$WORK_DIR/ts-prepare.log" appdb unset -f docker +# Older backup version that the destination image still ships: restore in +# place, no compatibility image, container or volume. +printf 'appdb\tappuser\t1\tdb-001.sql\t2.27.2\n' >"$TS_PREP_DIR/manifest.tsv" +MOCK_COMPAT_PULL=false +MOCK_HAS_LIBRARY=true +MOCK_IMAGE_RM=false +# Mock docker CLI for a 2.31.0 destination that lists 2.27.2. +docker() { + case "$*" in + *"default_version FROM pg_available_extensions"*) printf '2.31.0\n' ;; + *"SHOW server_version_num"*) printf '170010\n' ;; + *"FROM pg_available_extension_versions"*"'2.27.2'"*) printf '1\n' ;; + *"FROM pg_available_extension_versions"*) printf '0\n' ;; + *"pkglibdir"*) [ "$MOCK_HAS_LIBRARY" = true ] ;; + *" pull "* | "pull "*) MOCK_COMPAT_PULL=true; return 1 ;; + "image rm "*) MOCK_IMAGE_RM=true ;; + "info "*) printf '/var/lib/docker\n' ;; + *) return 1 ;; + esac +} +assert_true "ts_installable: listed version with its library" \ + timescaledb_version_installable destination appuser pass 2.27.2 "$WORK_DIR/ts-prepare.log" +assert_false "ts_installable: unlisted version" \ + timescaledb_version_installable destination appuser pass 2.16.1 "$WORK_DIR/ts-prepare.log" +assert_false "ts_installable: unsafe version is never queried" \ + timescaledb_version_installable destination appuser pass "2.27'; --" "$WORK_DIR/ts-prepare.log" +assert_true "ts_prepare: shipped older version restores in place" \ + pg_prepare_timescaledb_compatible_dumps destination appuser pass "$TS_PREP_DIR" \ + "$WORK_DIR/ts-prepare/output/pg_dump" "$WORK_DIR/ts-prepare.log" appdb +assert_eq "$PG_PREPARED_DUMP_DIR" "$TS_PREP_DIR" "ts_prepare: in-place restore keeps the original dump" +assert_eq "$MOCK_COMPAT_PULL" "false" "ts_prepare: in-place restore pulls no compatibility image" + +# A listed version without its library falls back to the conversion, which +# refuses before pulling anything when the disk is too small. +MOCK_HAS_LIBRARY=false +assert_false "ts_installable: listed version without its library" \ + timescaledb_version_installable destination appuser pass 2.27.2 "$WORK_DIR/ts-prepare.log" +_real_free_space_kb=$(declare -f free_space_kb) +free_space_kb() { printf '1024\n'; } +assert_false "ts_prepare: conversion refused when disk space is short" \ + pg_prepare_timescaledb_compatible_dumps destination appuser pass "$TS_PREP_DIR" \ + "$WORK_DIR/ts-prepare/output/pg_dump" "$WORK_DIR/ts-prepare.log" appdb +assert_eq "$MOCK_COMPAT_PULL" "false" "ts_prepare: short disk stops before the image pull" + +# With enough space the image is pulled; a failed pull still removes it. +free_space_kb() { printf '%s\n' $((100 * 1048576)); } +assert_false "ts_prepare: failed compatibility pull fails closed" \ + pg_prepare_timescaledb_compatible_dumps destination appuser pass "$TS_PREP_DIR" \ + "$WORK_DIR/ts-prepare/output/pg_dump" "$WORK_DIR/ts-prepare.log" appdb +assert_eq "$MOCK_COMPAT_PULL" "true" "ts_prepare: missing library triggers the conversion" +assert_eq "$MOCK_IMAGE_RM" "true" "ts_prepare: image pulled by the restore is removed" +assert_eq "$TIMESCALEDB_COMPAT_PULLED_IMAGE" "" "ts_prepare: pulled image marker cleared" +unset -f docker +eval "$_real_free_space_kb" + # ----------------------------------------------------------------------- # format_timescaledb_mismatch_help # ----------------------------------------------------------------------- From 9b4bc05e35f9a049f812bd8762977a762946c1b8 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 2 Nov 2026 16:34:00 +0330 Subject: [PATCH 2/3] fix(restore): address review on the TimescaleDB in-place restore - Require an update path from the backup's version to the server's version (pg_extension_update_paths) before choosing the in-place path. - When Docker storage and the restore staging dir share a filesystem, require their combined space estimate before pulling the image. - Arm the interrupt cleanup before the image pull, and track the container before docker run, so Ctrl-C at any point removes the image, volume and container. --- lib/pasarguard-restore.sh | 46 ++++++++++++++++++++++++++++----------- tests/unit_pasarguard.sh | 24 ++++++++++++++++---- 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/lib/pasarguard-restore.sh b/lib/pasarguard-restore.sh index 77bf18b..fd3ee11 100644 --- a/lib/pasarguard-restore.sh +++ b/lib/pasarguard-restore.sh @@ -192,19 +192,27 @@ timescaledb_version_is_safe() { } # True (0) when the destination container can create the timescaledb extension -# at exactly this version. The stock image ships the shared libraries of many -# older releases next to the current one. PostgreSQL also lists versions whose -# library is missing, and trying one of those kills the session, so both the -# install script and the library are checked without touching any database. +# at exactly this version and update it to target_version. The stock image +# ships the shared libraries of many older releases next to the current one. +# PostgreSQL also lists versions whose library is missing, and trying one of +# those kills the session, so the install script, the update path and the +# library are all checked without touching any database. timescaledb_version_installable() { - local container="$1" admin_user="$2" admin_password="$3" version="$4" log_file="$5" + local container="$1" admin_user="$2" admin_password="$3" version="$4" log_file="$5" target_version="$6" timescaledb_version_is_safe "$version" || return 1 + timescaledb_version_is_safe "$target_version" || return 1 local listed="" listed=$(docker exec -e PGPASSWORD="$admin_password" "$container" \ psql -X -U "$admin_user" -d postgres -At \ -c "SELECT count(*) FROM pg_available_extension_versions WHERE name = 'timescaledb' AND version = '$version';" \ 2>>"$log_file") || return 1 [ "$listed" = "1" ] || return 1 + local update_path="" + update_path=$(docker exec -e PGPASSWORD="$admin_password" "$container" \ + psql -X -U "$admin_user" -d postgres -At \ + -c "SELECT count(*) FROM pg_extension_update_paths('timescaledb') WHERE source = '$version' AND target = '$target_version' AND path IS NOT NULL;" \ + 2>>"$log_file") || return 1 + [ "$update_path" = "1" ] || return 1 # shellcheck disable=SC2016 # $1 expands inside the container shell. docker exec "$container" sh -c 'test -f "$(pg_config --pkglibdir)/timescaledb-$1.so"' sh "$version" 2>>"$log_file" } @@ -214,6 +222,11 @@ free_space_kb() { df -Pk "$1" 2>/dev/null | awk 'NR == 2 {print $4}' } +# Print the device id of the filesystem that holds a path. +filesystem_id() { + stat -c %d "$1" 2>/dev/null +} + # The compatibility image is several GB once unpacked. Remove it after use when # this restore pulled it, so a one-off conversion does not keep using the disk. remove_pulled_timescaledb_compat_image() { @@ -340,7 +353,7 @@ pg_prepare_timescaledb_compatible_dumps() { [ -n "$dbname" ] || continue if [ "$has_ts" = "1" ] && [ -n "$source_version" ] && [ "$source_version" != "$target_version" ]; then if timescaledb_version_installable "$destination_container" "$admin_user" "$admin_password" \ - "$source_version" "$log_file"; then + "$source_version" "$log_file" "$target_version"; then in_place_versions+=" $source_version" continue fi @@ -396,6 +409,11 @@ pg_prepare_timescaledb_compatible_dumps() { docker_root=$(docker info -f '{{.DockerRootDir}}' 2>>"$log_file") || docker_root="" [ -n "$docker_root" ] && docker_free_kb=$(free_space_kb "$docker_root") staging_free_kb=$(free_space_kb "$(dirname "$output_dir")") + # Docker storage and the restore staging dir are often the same disk. Then + # both estimates come out of the same free space. + if [ -n "$docker_root" ] && [ "$(filesystem_id "$docker_root")" = "$(filesystem_id "$(dirname "$output_dir")")" ]; then + docker_need_kb=$((docker_need_kb + dump_kb)) + fi if [[ "$docker_free_kb" =~ ^[0-9]+$ ]] && [ "$docker_free_kb" -lt "$docker_need_kb" ]; then colorized_echo red "Not enough disk space for the TimescaleDB compatibility conversion: it needs about $((docker_need_kb / 1048576 + 1)) GB free under $docker_root, and $((docker_free_kb / 1048576)) GB is free." echo "Compatibility conversion needs ${docker_need_kb} KiB under $docker_root; ${docker_free_kb} KiB free" >>"$log_file" @@ -409,20 +427,25 @@ pg_prepare_timescaledb_compatible_dumps() { colorized_echo blue "Preparing TimescaleDB $target_version-compatible dumps before changing the destination..." colorized_echo yellow "This server does not ship the backup's TimescaleDB version, so a temporary conversion container is used. It is removed when the restore finishes." + # Arm cleanup before the pull so an interrupt also removes the image. + arm_timescaledb_compat_cleanup "" "" if [ "$image_present" = false ]; then colorized_echo blue "Pulling temporary compatibility image: $compat_image" TIMESCALEDB_COMPAT_PULLED_IMAGE="$compat_image" if ! docker pull "$compat_image" >>"$log_file" 2>&1; then echo "Could not pull TimescaleDB compatibility image: $compat_image" >>"$log_file" - remove_pulled_timescaledb_compat_image + finish_timescaledb_compat_cleanup return 1 fi fi if ! compat_volume=$(docker volume create --label "com.pasarguard.restore=$compat_suffix" 2>>"$log_file"); then echo "Could not create temporary TimescaleDB compatibility volume" >>"$log_file" - remove_pulled_timescaledb_compat_image + finish_timescaledb_compat_cleanup return 1 fi + # Track the container before it exists: docker rm -f on a missing name is + # harmless, and an interrupt during docker run must not leave it behind. + arm_timescaledb_compat_cleanup "$compat_container" "$compat_volume" if ! docker run -d --name "$compat_container" --restart=no \ --user 0 \ -e POSTGRES_USER="$admin_user" \ @@ -433,11 +456,9 @@ pg_prepare_timescaledb_compatible_dumps() { -v "$compat_volume:$compat_pgdata" \ "$compat_image" >>"$log_file" 2>&1; then echo "Could not start temporary TimescaleDB compatibility container" >>"$log_file" - cleanup_timescaledb_compat_container "$compat_container" "$compat_volume" - remove_pulled_timescaledb_compat_image + finish_timescaledb_compat_cleanup return 1 fi - arm_timescaledb_compat_cleanup "$compat_container" "$compat_volume" local ready=false local waited=0 @@ -710,8 +731,7 @@ pg_restore_all_user_databases() { target_ts="" fi if [ "$probe_ok" = "1" ] && ! timescaledb_version_matches "$ts_version" "$target_ts" && - timescaledb_version_is_safe "$target_ts" && - timescaledb_version_installable "$container_name" "$admin_user" "$admin_password" "$ts_version" "$log_file"; then + timescaledb_version_installable "$container_name" "$admin_user" "$admin_password" "$ts_version" "$log_file" "$target_ts"; then update_extension_to="$target_ts" elif [ "$probe_ok" = "1" ] && ! timescaledb_version_matches "$ts_version" "$target_ts"; then local svn="" pg_major="" diff --git a/tests/unit_pasarguard.sh b/tests/unit_pasarguard.sh index f5077ae..89a0cab 100644 --- a/tests/unit_pasarguard.sh +++ b/tests/unit_pasarguard.sh @@ -707,6 +707,8 @@ docker() { *"SHOW server_version_num"*) printf '170010\n' ;; *"FROM pg_available_extension_versions"*"'2.27.2'"*) printf '1\n' ;; *"FROM pg_available_extension_versions"*) printf '0\n' ;; + *"pg_extension_update_paths"*"'2.27.2'"*"'2.31.0'"*) printf '1\n' ;; + *"pg_extension_update_paths"*) printf '0\n' ;; *"pkglibdir"*) [ "$MOCK_HAS_LIBRARY" = true ] ;; *" pull "* | "pull "*) MOCK_COMPAT_PULL=true; return 1 ;; "image rm "*) MOCK_IMAGE_RM=true ;; @@ -715,11 +717,13 @@ docker() { esac } assert_true "ts_installable: listed version with its library" \ - timescaledb_version_installable destination appuser pass 2.27.2 "$WORK_DIR/ts-prepare.log" + timescaledb_version_installable destination appuser pass 2.27.2 "$WORK_DIR/ts-prepare.log" 2.31.0 +assert_false "ts_installable: no update path to the target" \ + timescaledb_version_installable destination appuser pass 2.27.2 "$WORK_DIR/ts-prepare.log" 2.30.0 assert_false "ts_installable: unlisted version" \ - timescaledb_version_installable destination appuser pass 2.16.1 "$WORK_DIR/ts-prepare.log" + timescaledb_version_installable destination appuser pass 2.16.1 "$WORK_DIR/ts-prepare.log" 2.31.0 assert_false "ts_installable: unsafe version is never queried" \ - timescaledb_version_installable destination appuser pass "2.27'; --" "$WORK_DIR/ts-prepare.log" + timescaledb_version_installable destination appuser pass "2.27'; --" "$WORK_DIR/ts-prepare.log" 2.31.0 assert_true "ts_prepare: shipped older version restores in place" \ pg_prepare_timescaledb_compatible_dumps destination appuser pass "$TS_PREP_DIR" \ "$WORK_DIR/ts-prepare/output/pg_dump" "$WORK_DIR/ts-prepare.log" appdb @@ -730,7 +734,7 @@ assert_eq "$MOCK_COMPAT_PULL" "false" "ts_prepare: in-place restore pulls no com # refuses before pulling anything when the disk is too small. MOCK_HAS_LIBRARY=false assert_false "ts_installable: listed version without its library" \ - timescaledb_version_installable destination appuser pass 2.27.2 "$WORK_DIR/ts-prepare.log" + timescaledb_version_installable destination appuser pass 2.27.2 "$WORK_DIR/ts-prepare.log" 2.31.0 _real_free_space_kb=$(declare -f free_space_kb) free_space_kb() { printf '1024\n'; } assert_false "ts_prepare: conversion refused when disk space is short" \ @@ -738,6 +742,18 @@ assert_false "ts_prepare: conversion refused when disk space is short" \ "$WORK_DIR/ts-prepare/output/pg_dump" "$WORK_DIR/ts-prepare.log" appdb assert_eq "$MOCK_COMPAT_PULL" "false" "ts_prepare: short disk stops before the image pull" +# Enough space for Docker or for the staging dumps alone, but not for both on +# one shared filesystem. +_ts_dump_kb=$(du -sk "$TS_PREP_DIR" | awk '{print $1}') +_real_filesystem_id=$(declare -f filesystem_id) +filesystem_id() { printf 'shared\n'; } +free_space_kb() { printf '%s\n' $((TIMESCALEDB_COMPAT_IMAGE_KB + _ts_dump_kb * 3 - 1)); } +assert_false "ts_prepare: shared filesystem needs Docker and staging space together" \ + pg_prepare_timescaledb_compatible_dumps destination appuser pass "$TS_PREP_DIR" \ + "$WORK_DIR/ts-prepare/output/pg_dump" "$WORK_DIR/ts-prepare.log" appdb +assert_eq "$MOCK_COMPAT_PULL" "false" "ts_prepare: shared filesystem shortage stops before the pull" +eval "$_real_filesystem_id" + # With enough space the image is pulled; a failed pull still removes it. free_space_kb() { printf '%s\n' $((100 * 1048576)); } assert_false "ts_prepare: failed compatibility pull fails closed" \ From 0000fe1ba1f342887e76f68d958ca50009eeaf28 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 2 Nov 2026 16:57:17 +0330 Subject: [PATCH 3/3] fix(restore): stop the TimescaleDB conversion when free space cannot be measured --- lib/pasarguard-restore.sh | 11 +++++++++-- tests/unit_pasarguard.sh | 11 ++++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/lib/pasarguard-restore.sh b/lib/pasarguard-restore.sh index fd3ee11..232b902 100644 --- a/lib/pasarguard-restore.sh +++ b/lib/pasarguard-restore.sh @@ -414,12 +414,19 @@ pg_prepare_timescaledb_compatible_dumps() { if [ -n "$docker_root" ] && [ "$(filesystem_id "$docker_root")" = "$(filesystem_id "$(dirname "$output_dir")")" ]; then docker_need_kb=$((docker_need_kb + dump_kb)) fi - if [[ "$docker_free_kb" =~ ^[0-9]+$ ]] && [ "$docker_free_kb" -lt "$docker_need_kb" ]; then + # Without a reading the conversion could fill the disk the live database + # uses, so an unknown amount of free space stops it as well. + if [[ ! "$docker_free_kb" =~ ^[0-9]+$ ]] || [[ ! "$staging_free_kb" =~ ^[0-9]+$ ]]; then + colorized_echo red "Could not measure free disk space for the TimescaleDB compatibility conversion (Docker root: ${docker_root:-unknown}). It needs about $((docker_need_kb / 1048576 + 1)) GB free; the current database was not changed." + echo "Free space unknown: docker_root='${docker_root}' docker_free_kb='${docker_free_kb}' staging_free_kb='${staging_free_kb}'" >>"$log_file" + return 1 + fi + if [ "$docker_free_kb" -lt "$docker_need_kb" ]; then colorized_echo red "Not enough disk space for the TimescaleDB compatibility conversion: it needs about $((docker_need_kb / 1048576 + 1)) GB free under $docker_root, and $((docker_free_kb / 1048576)) GB is free." echo "Compatibility conversion needs ${docker_need_kb} KiB under $docker_root; ${docker_free_kb} KiB free" >>"$log_file" return 1 fi - if [[ "$staging_free_kb" =~ ^[0-9]+$ ]] && [ "$staging_free_kb" -lt "$dump_kb" ]; then + if [ "$staging_free_kb" -lt "$dump_kb" ]; then colorized_echo red "Not enough disk space for the converted TimescaleDB dumps: they need about $((dump_kb / 1048576 + 1)) GB free next to the restore files." echo "Converted dumps need ${dump_kb} KiB in $(dirname "$output_dir"); ${staging_free_kb} KiB free" >>"$log_file" return 1 diff --git a/tests/unit_pasarguard.sh b/tests/unit_pasarguard.sh index 89a0cab..5a3f364 100644 --- a/tests/unit_pasarguard.sh +++ b/tests/unit_pasarguard.sh @@ -700,6 +700,7 @@ printf 'appdb\tappuser\t1\tdb-001.sql\t2.27.2\n' >"$TS_PREP_DIR/manifest.tsv" MOCK_COMPAT_PULL=false MOCK_HAS_LIBRARY=true MOCK_IMAGE_RM=false +MOCK_DOCKER_INFO_FAILS=false # Mock docker CLI for a 2.31.0 destination that lists 2.27.2. docker() { case "$*" in @@ -712,7 +713,7 @@ docker() { *"pkglibdir"*) [ "$MOCK_HAS_LIBRARY" = true ] ;; *" pull "* | "pull "*) MOCK_COMPAT_PULL=true; return 1 ;; "image rm "*) MOCK_IMAGE_RM=true ;; - "info "*) printf '/var/lib/docker\n' ;; + "info "*) [ "$MOCK_DOCKER_INFO_FAILS" = true ] && return 1; printf '/var/lib/docker\n' ;; *) return 1 ;; esac } @@ -754,6 +755,14 @@ assert_false "ts_prepare: shared filesystem needs Docker and staging space toget assert_eq "$MOCK_COMPAT_PULL" "false" "ts_prepare: shared filesystem shortage stops before the pull" eval "$_real_filesystem_id" +# An unknown Docker root means free space cannot be measured: stop, no pull. +MOCK_DOCKER_INFO_FAILS=true +assert_false "ts_prepare: unmeasurable Docker free space fails closed" \ + pg_prepare_timescaledb_compatible_dumps destination appuser pass "$TS_PREP_DIR" \ + "$WORK_DIR/ts-prepare/output/pg_dump" "$WORK_DIR/ts-prepare.log" appdb +assert_eq "$MOCK_COMPAT_PULL" "false" "ts_prepare: unmeasurable free space stops before the pull" +MOCK_DOCKER_INFO_FAILS=false + # With enough space the image is pulled; a failed pull still removes it. free_space_kb() { printf '%s\n' $((100 * 1048576)); } assert_false "ts_prepare: failed compatibility pull fails closed" \