diff --git a/app/Console/Commands/SyncBunny.php b/app/Console/Commands/SyncBunny.php index d6d77f22e..3f3e213fd 100644 --- a/app/Console/Commands/SyncBunny.php +++ b/app/Console/Commands/SyncBunny.php @@ -44,6 +44,7 @@ class SyncBunny extends Command $compose_file_prod = 'docker-compose.prod.yml'; $install_script = 'install.sh'; $upgrade_script = 'upgrade.sh'; + $upgrade_postgres_script = 'upgrade-postgres.sh'; $production_env = '.env.production'; $service_template = config('constants.services.file_name'); $versions = 'versions.json'; @@ -52,6 +53,7 @@ class SyncBunny extends Command $compose_file_prod_location = "$parent_dir/$compose_file_prod"; $install_script_location = "$parent_dir/scripts/install.sh"; $upgrade_script_location = "$parent_dir/scripts/upgrade.sh"; + $upgrade_postgres_script_location = "$parent_dir/scripts/upgrade-postgres.sh"; $production_env_location = "$parent_dir/.env.production"; $versions_location = "$parent_dir/$versions"; @@ -87,6 +89,7 @@ class SyncBunny extends Command $compose_file_prod_location = "$parent_dir/other/nightly/$compose_file_prod"; $production_env_location = "$parent_dir/other/nightly/$production_env"; $upgrade_script_location = "$parent_dir/other/nightly/$upgrade_script"; + $upgrade_postgres_script_location = "$parent_dir/other/nightly/$upgrade_postgres_script"; $install_script_location = "$parent_dir/other/nightly/$install_script"; $versions_location = "$parent_dir/other/nightly/$versions"; } @@ -101,6 +104,7 @@ class SyncBunny extends Command $compose_file_prod_location => "$bunny_cdn/$bunny_cdn_path/$compose_file_prod", $production_env_location => "$bunny_cdn/$bunny_cdn_path/$production_env", $upgrade_script_location => "$bunny_cdn/$bunny_cdn_path/$upgrade_script", + $upgrade_postgres_script_location => "$bunny_cdn/$bunny_cdn_path/$upgrade_postgres_script", $install_script_location => "$bunny_cdn/$bunny_cdn_path/$install_script", ]; @@ -215,6 +219,7 @@ class SyncBunny extends Command $pool->storage(fileName: "$compose_file_prod_location")->put("/$bunny_cdn_storage_name/$bunny_cdn_path/$compose_file_prod"), $pool->storage(fileName: "$production_env_location")->put("/$bunny_cdn_storage_name/$bunny_cdn_path/$production_env"), $pool->storage(fileName: "$upgrade_script_location")->put("/$bunny_cdn_storage_name/$bunny_cdn_path/$upgrade_script"), + $pool->storage(fileName: "$upgrade_postgres_script_location")->put("/$bunny_cdn_storage_name/$bunny_cdn_path/$upgrade_postgres_script"), $pool->storage(fileName: "$install_script_location")->put("/$bunny_cdn_storage_name/$bunny_cdn_path/$install_script"), ]); Http::pool(fn (Pool $pool) => [ @@ -222,6 +227,7 @@ class SyncBunny extends Command $pool->purge("$bunny_cdn/$bunny_cdn_path/$compose_file_prod"), $pool->purge("$bunny_cdn/$bunny_cdn_path/$production_env"), $pool->purge("$bunny_cdn/$bunny_cdn_path/$upgrade_script"), + $pool->purge("$bunny_cdn/$bunny_cdn_path/$upgrade_postgres_script"), $pool->purge("$bunny_cdn/$bunny_cdn_path/$install_script"), ]); $this->info('All files uploaded & purged to BunnyCDN.'); diff --git a/other/nightly/install.sh b/other/nightly/install.sh index 4a91f7b1b..028652d80 100755 --- a/other/nightly/install.sh +++ b/other/nightly/install.sh @@ -781,10 +781,12 @@ curl -fsSL -L $CDN/.env.production -o /data/coolify/source/.env.production & PID3=$! curl -fsSL -L $CDN/upgrade.sh -o /data/coolify/source/upgrade.sh & PID4=$! +curl -fsSL -L $CDN/upgrade-postgres.sh -o /data/coolify/source/upgrade-postgres.sh & +PID5=$! # Wait for all downloads to complete and check for errors DOWNLOAD_FAILED=false -for PID in $PID1 $PID2 $PID3 $PID4; do +for PID in $PID1 $PID2 $PID3 $PID4 $PID5; do if ! wait $PID; then DOWNLOAD_FAILED=true fi @@ -795,6 +797,7 @@ if [ "$DOWNLOAD_FAILED" = true ]; then exit 1 fi +chmod +x /data/coolify/source/upgrade.sh /data/coolify/source/upgrade-postgres.sh log "All configuration files downloaded successfully" echo " Done." diff --git a/other/nightly/upgrade-postgres.sh b/other/nightly/upgrade-postgres.sh new file mode 100755 index 000000000..e9e12a5fe --- /dev/null +++ b/other/nightly/upgrade-postgres.sh @@ -0,0 +1,381 @@ +#!/bin/bash +## Explicit Coolify internal PostgreSQL major-version migrator. +## This script is intentionally not run by upgrade.sh automatically. + +set -Eeuo pipefail + +SOURCE_DIR="/data/coolify/source" +ENV_FILE="${SOURCE_DIR}/.env" +BACKUP_DIR="/data/coolify/backups/internal-postgres" +OVERRIDE_FILE="${SOURCE_DIR}/docker-compose.postgres-upgrade.yml" +ROLLBACK_FILE="${SOURCE_DIR}/postgres-upgrade-rollback.env" +DATE=$(date +%Y-%m-%d-%H-%M-%S) +LOGFILE="${SOURCE_DIR}/postgres-upgrade-${DATE}.log" +COMMAND="${1:-upgrade}" +TARGET_MAJOR="${1:-18}" + +if [ "$COMMAND" = "rollback" ]; then + TARGET_MAJOR="" +else + COMMAND="upgrade" +fi + +TARGET_IMAGE="${COOLIFY_POSTGRES_TARGET_IMAGE:-postgres:${TARGET_MAJOR}-alpine}" +TARGET_VOLUME="${COOLIFY_POSTGRES_TARGET_VOLUME:-coolify-db-pg${TARGET_MAJOR}}" +TEMP_CONTAINER="coolify-db-pg${TARGET_MAJOR:-rollback}-restore-${DATE}" +DUMP_FILE="${BACKUP_DIR}/postgres-upgrade-${DATE}.sql.gz" + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOGFILE" +} + +fail() { + log "ERROR: $1" + exit 1 +} + +usage() { + cat < + $0 rollback + +Examples: + $0 18 + $0 rollback + +Environment overrides: + COOLIFY_POSTGRES_TARGET_IMAGE=postgres:18-alpine + COOLIFY_POSTGRES_TARGET_VOLUME=coolify-db-pg18 +EOF +} + +cleanup() { + docker rm -f "$TEMP_CONTAINER" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +get_env_var() { + local key="$1" + local fallback="${2:-}" + local value + + value=$(grep -E "^${key}=" "$ENV_FILE" 2>/dev/null | tail -n 1 | cut -d '=' -f 2- || true) + value="${value%\"}" + value="${value#\"}" + value="${value%\'}" + value="${value#\'}" + + if [ -z "$value" ]; then + printf '%s' "$fallback" + else + printf '%s' "$value" + fi +} + +wait_for_postgres() { + local container="$1" + local user="$2" + local database="$3" + local attempts=60 + + for _ in $(seq 1 "$attempts"); do + if docker exec "$container" pg_isready -U "$user" -d "$database" >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + + return 1 +} + +compose_files() { + printf -- '-f %s/docker-compose.yml -f %s/docker-compose.prod.yml ' "$SOURCE_DIR" "$SOURCE_DIR" + + if [ -f "${SOURCE_DIR}/docker-compose.custom.yml" ]; then + printf -- '-f %s/docker-compose.custom.yml ' "$SOURCE_DIR" + fi + + if [ -f "$OVERRIDE_FILE" ]; then + printf -- '-f %s ' "$OVERRIDE_FILE" + fi +} + +validate_target_major() { + case "$TARGET_MAJOR" in + ''|*[!0-9]*) + usage + fail "Target major version must be numeric. Example: $0 18" + ;; + esac + + if [ "$TARGET_MAJOR" -lt 10 ]; then + fail "Target major version must be 10 or higher." + fi +} + +mount_path_for_major() { + local major="$1" + + if [ "$major" -ge 18 ]; then + printf '%s' '/var/lib/postgresql' + else + printf '%s' '/var/lib/postgresql/data' + fi +} + +current_postgres_mount_name() { + docker inspect coolify-db --format '{{range .Mounts}}{{if or (eq .Destination "/var/lib/postgresql/data") (eq .Destination "/var/lib/postgresql")}}{{.Name}}{{end}}{{end}}' 2>/dev/null +} + +current_postgres_mount_path() { + docker inspect coolify-db --format '{{range .Mounts}}{{if or (eq .Destination "/var/lib/postgresql/data") (eq .Destination "/var/lib/postgresql")}}{{.Destination}}{{end}}{{end}}' 2>/dev/null +} + +current_postgres_image() { + docker inspect coolify-db --format '{{.Config.Image}}' 2>/dev/null +} + +write_override_file() { + local image="$1" + local volume="$2" + local mount_path="$3" + + cat > "$OVERRIDE_FILE" < "$ROLLBACK_FILE" </dev/null 2>&1 || fail "Docker is required." + docker info >/dev/null 2>&1 || fail "Docker daemon is not reachable." +} + +rollback_postgres() { + validate_common_requirements + + [ -f "$ROLLBACK_FILE" ] || fail "Missing rollback metadata file: ${ROLLBACK_FILE}" + + # shellcheck disable=SC1090 + . "$ROLLBACK_FILE" + + [ -n "${PREVIOUS_IMAGE:-}" ] || fail "Rollback metadata is missing PREVIOUS_IMAGE." + [ -n "${PREVIOUS_VOLUME:-}" ] || fail "Rollback metadata is missing PREVIOUS_VOLUME." + [ -n "${PREVIOUS_MOUNT_PATH:-}" ] || fail "Rollback metadata is missing PREVIOUS_MOUNT_PATH." + [ -n "${PREVIOUS_OVERRIDE_PRESENT:-}" ] || fail "Rollback metadata is missing PREVIOUS_OVERRIDE_PRESENT." + + log "Rolling back Coolify internal PostgreSQL." + log "Previous image: ${PREVIOUS_IMAGE}" + log "Previous volume: ${PREVIOUS_VOLUME}" + log "Previous mount path: ${PREVIOUS_MOUNT_PATH}" + + docker volume inspect "$PREVIOUS_VOLUME" >/dev/null 2>&1 || fail "Previous volume '${PREVIOUS_VOLUME}' does not exist." + + log "Stopping Coolify application container before rollback." + docker stop coolify >>"$LOGFILE" 2>&1 || true + + log "Removing current coolify-db container. Current upgraded volume is kept untouched." + docker rm -f coolify-db >>"$LOGFILE" 2>&1 || true + + if [ "$PREVIOUS_OVERRIDE_PRESENT" = "true" ]; then + log "Restoring previous PostgreSQL compose override." + write_override_file "$PREVIOUS_IMAGE" "$PREVIOUS_VOLUME" "$PREVIOUS_MOUNT_PATH" + else + log "Removing PostgreSQL compose override to restore base compose configuration." + rm -f "$OVERRIDE_FILE" + fi + + log "Starting Coolify stack with rollback database volume." + start_stack >>"$LOGFILE" 2>&1 || fail "Could not start Coolify stack after rollback. See ${LOGFILE}." + + log "Rollback completed successfully." + cat <>"$LOGFILE" 2>&1 || fail "Could not start coolify-db." + fi + + wait_for_postgres coolify-db "$DB_USERNAME" "$DB_DATABASE" || fail "Existing coolify-db is not ready." + + SERVER_VERSION_NUM=$(docker exec coolify-db psql -U "$DB_USERNAME" -d "$DB_DATABASE" -Atc 'SHOW server_version_num;' | tr -d '[:space:]') + CURRENT_MAJOR=$((SERVER_VERSION_NUM / 10000)) + PREVIOUS_VOLUME=$(current_postgres_mount_name) + PREVIOUS_MOUNT_PATH=$(current_postgres_mount_path) + PREVIOUS_IMAGE=$(current_postgres_image) + + [ -n "$PREVIOUS_VOLUME" ] || fail "Could not detect current PostgreSQL Docker volume." + [ -n "$PREVIOUS_MOUNT_PATH" ] || fail "Could not detect current PostgreSQL mount path." + [ -n "$PREVIOUS_IMAGE" ] || fail "Could not detect current PostgreSQL image." + + if [ -f "$OVERRIDE_FILE" ]; then + PREVIOUS_OVERRIDE_PRESENT=true + else + PREVIOUS_OVERRIDE_PRESENT=false + fi + + log "Current PostgreSQL major: ${CURRENT_MAJOR}" + log "Current active volume: ${PREVIOUS_VOLUME}" + log "Current image: ${PREVIOUS_IMAGE}" + log "Current mount path: ${PREVIOUS_MOUNT_PATH}" + + if [ "$CURRENT_MAJOR" -eq "$TARGET_MAJOR" ]; then + log "PostgreSQL is already on major ${TARGET_MAJOR}. Nothing to do." + exit 0 + fi + + if [ "$CURRENT_MAJOR" -gt "$TARGET_MAJOR" ]; then + fail "Downgrade from ${CURRENT_MAJOR} to ${TARGET_MAJOR} is not supported. Use '$0 rollback' to restore the previous upgrade state." + fi + + if docker volume inspect "$TARGET_VOLUME" >/dev/null 2>&1; then + fail "Target volume '${TARGET_VOLUME}' already exists. Set COOLIFY_POSTGRES_TARGET_VOLUME to a new name or remove the old failed target volume." + fi + + log "Stopping Coolify application container to prevent writes during dump." + docker stop coolify >>"$LOGFILE" 2>&1 || true + + log "Creating compressed dump at ${DUMP_FILE}." + docker exec coolify-db pg_dumpall -U "$DB_USERNAME" | gzip -c > "$DUMP_FILE" + chmod 600 "$DUMP_FILE" + + if [ ! -s "$DUMP_FILE" ]; then + fail "Dump file is empty. Aborting." + fi + + log "Creating target Docker volume '${TARGET_VOLUME}'." + docker volume create "$TARGET_VOLUME" >>"$LOGFILE" 2>&1 + + log "Pulling ${TARGET_IMAGE}." + docker pull "$TARGET_IMAGE" >>"$LOGFILE" 2>&1 + + log "Starting temporary PostgreSQL ${TARGET_MAJOR} container." + docker run -d \ + --name "$TEMP_CONTAINER" \ + --network coolify \ + -e POSTGRES_HOST_AUTH_METHOD=trust \ + -v "${TARGET_VOLUME}:${TARGET_MOUNT_PATH}" \ + "$TARGET_IMAGE" >>"$LOGFILE" 2>&1 + + wait_for_postgres "$TEMP_CONTAINER" postgres postgres || fail "Temporary PostgreSQL ${TARGET_MAJOR} container did not become ready." + + log "Restoring dump into target volume." + gunzip -c "$DUMP_FILE" | docker exec -i "$TEMP_CONTAINER" psql -U postgres -d postgres >>"$LOGFILE" 2>&1 + + log "Smoke-checking restored Coolify database." + docker exec "$TEMP_CONTAINER" psql -U "$DB_USERNAME" -d "$DB_DATABASE" -Atc 'SELECT 1;' | grep -qx '1' || fail "Restored database smoke check failed." + + log "Saving rollback metadata to ${ROLLBACK_FILE}." + write_rollback_file "$PREVIOUS_IMAGE" "$PREVIOUS_VOLUME" "$PREVIOUS_MOUNT_PATH" "$PREVIOUS_OVERRIDE_PRESENT" "$TARGET_IMAGE" "$TARGET_VOLUME" + + log "Writing Docker Compose override to ${OVERRIDE_FILE}." + write_override_file "$TARGET_IMAGE" "$TARGET_VOLUME" "$TARGET_MOUNT_PATH" + + log "Stopping temporary restore container." + docker rm -f "$TEMP_CONTAINER" >>"$LOGFILE" 2>&1 || true + + log "Stopping old coolify-db container. Previous volume '${PREVIOUS_VOLUME}' will be kept for rollback." + docker rm -f coolify-db >>"$LOGFILE" 2>&1 || true + + log "Starting Coolify stack with PostgreSQL ${TARGET_MAJOR}." + start_stack >>"$LOGFILE" 2>&1 || fail "Could not start Coolify stack with upgraded PostgreSQL. See ${LOGFILE}." + + log "Coolify internal PostgreSQL upgrade completed successfully." + print_rollback_instructions +} + +case "$COMMAND" in + rollback) + rollback_postgres + ;; + upgrade) + upgrade_postgres + ;; + -h|--help|help) + usage + ;; + *) + usage + fail "Unknown command: ${COMMAND}" + ;; +esac diff --git a/other/nightly/upgrade.sh b/other/nightly/upgrade.sh index a21d39e41..c8fb9a98f 100644 --- a/other/nightly/upgrade.sh +++ b/other/nightly/upgrade.sh @@ -56,6 +56,9 @@ log "Downloading docker-compose.prod.yml from ${CDN}/docker-compose.prod.yml" curl -fsSL -L $CDN/docker-compose.prod.yml -o /data/coolify/source/docker-compose.prod.yml log "Downloading .env.production from ${CDN}/.env.production" curl -fsSL -L $CDN/.env.production -o /data/coolify/source/.env.production +log "Downloading upgrade-postgres.sh from ${CDN}/upgrade-postgres.sh" +curl -fsSL -L $CDN/upgrade-postgres.sh -o /data/coolify/source/upgrade-postgres.sh +chmod +x /data/coolify/source/upgrade-postgres.sh log "Configuration files downloaded successfully" echo " Done." @@ -69,6 +72,12 @@ if [ -f /data/coolify/source/docker-compose.custom.yml ]; then log "Including custom docker-compose.yml in image extraction" fi +# Check if PostgreSQL upgrade override exists +if [ -f /data/coolify/source/docker-compose.postgres-upgrade.yml ]; then + COMPOSE_FILES="$COMPOSE_FILES -f /data/coolify/source/docker-compose.postgres-upgrade.yml" + log "Including PostgreSQL upgrade compose override in image extraction" +fi + # Get all unique images from docker compose config # LATEST_IMAGE env var is needed for image substitution in compose files IMAGES=$(LATEST_IMAGE=${LATEST_IMAGE} docker compose --env-file "$ENV_FILE" $COMPOSE_FILES config --images 2>/dev/null | sort -u) @@ -236,15 +245,18 @@ nohup bash -c " echo '============================================================' >>\"\$LOGFILE\" write_status '5' 'Starting new containers' + COMPOSE_FILES='-f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml' if [ -f /data/coolify/source/docker-compose.custom.yml ]; then log 'Using custom docker-compose.yml' - log 'Running docker compose up with custom configuration...' - docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml -f /data/coolify/source/docker-compose.custom.yml up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1 - else - log 'Using standard docker-compose configuration' - log 'Running docker compose up...' - docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1 + COMPOSE_FILES="\$COMPOSE_FILES -f /data/coolify/source/docker-compose.custom.yml" fi + if [ -f /data/coolify/source/docker-compose.postgres-upgrade.yml ]; then + log 'Using PostgreSQL upgrade compose override' + COMPOSE_FILES="\$COMPOSE_FILES -f /data/coolify/source/docker-compose.postgres-upgrade.yml" + fi + + log 'Running docker compose up...' + docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1 log 'Docker compose up completed' # Final log entry diff --git a/scripts/install.sh b/scripts/install.sh index 15f172f6d..430dd9d83 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -781,10 +781,12 @@ curl -fsSL -L $CDN/.env.production -o /data/coolify/source/.env.production & PID3=$! curl -fsSL -L $CDN/upgrade.sh -o /data/coolify/source/upgrade.sh & PID4=$! +curl -fsSL -L $CDN/upgrade-postgres.sh -o /data/coolify/source/upgrade-postgres.sh & +PID5=$! # Wait for all downloads to complete and check for errors DOWNLOAD_FAILED=false -for PID in $PID1 $PID2 $PID3 $PID4; do +for PID in $PID1 $PID2 $PID3 $PID4 $PID5; do if ! wait $PID; then DOWNLOAD_FAILED=true fi @@ -795,6 +797,7 @@ if [ "$DOWNLOAD_FAILED" = true ]; then exit 1 fi +chmod +x /data/coolify/source/upgrade.sh /data/coolify/source/upgrade-postgres.sh log "All configuration files downloaded successfully" echo " Done." diff --git a/scripts/upgrade-postgres.sh b/scripts/upgrade-postgres.sh new file mode 100755 index 000000000..e9e12a5fe --- /dev/null +++ b/scripts/upgrade-postgres.sh @@ -0,0 +1,381 @@ +#!/bin/bash +## Explicit Coolify internal PostgreSQL major-version migrator. +## This script is intentionally not run by upgrade.sh automatically. + +set -Eeuo pipefail + +SOURCE_DIR="/data/coolify/source" +ENV_FILE="${SOURCE_DIR}/.env" +BACKUP_DIR="/data/coolify/backups/internal-postgres" +OVERRIDE_FILE="${SOURCE_DIR}/docker-compose.postgres-upgrade.yml" +ROLLBACK_FILE="${SOURCE_DIR}/postgres-upgrade-rollback.env" +DATE=$(date +%Y-%m-%d-%H-%M-%S) +LOGFILE="${SOURCE_DIR}/postgres-upgrade-${DATE}.log" +COMMAND="${1:-upgrade}" +TARGET_MAJOR="${1:-18}" + +if [ "$COMMAND" = "rollback" ]; then + TARGET_MAJOR="" +else + COMMAND="upgrade" +fi + +TARGET_IMAGE="${COOLIFY_POSTGRES_TARGET_IMAGE:-postgres:${TARGET_MAJOR}-alpine}" +TARGET_VOLUME="${COOLIFY_POSTGRES_TARGET_VOLUME:-coolify-db-pg${TARGET_MAJOR}}" +TEMP_CONTAINER="coolify-db-pg${TARGET_MAJOR:-rollback}-restore-${DATE}" +DUMP_FILE="${BACKUP_DIR}/postgres-upgrade-${DATE}.sql.gz" + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOGFILE" +} + +fail() { + log "ERROR: $1" + exit 1 +} + +usage() { + cat < + $0 rollback + +Examples: + $0 18 + $0 rollback + +Environment overrides: + COOLIFY_POSTGRES_TARGET_IMAGE=postgres:18-alpine + COOLIFY_POSTGRES_TARGET_VOLUME=coolify-db-pg18 +EOF +} + +cleanup() { + docker rm -f "$TEMP_CONTAINER" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +get_env_var() { + local key="$1" + local fallback="${2:-}" + local value + + value=$(grep -E "^${key}=" "$ENV_FILE" 2>/dev/null | tail -n 1 | cut -d '=' -f 2- || true) + value="${value%\"}" + value="${value#\"}" + value="${value%\'}" + value="${value#\'}" + + if [ -z "$value" ]; then + printf '%s' "$fallback" + else + printf '%s' "$value" + fi +} + +wait_for_postgres() { + local container="$1" + local user="$2" + local database="$3" + local attempts=60 + + for _ in $(seq 1 "$attempts"); do + if docker exec "$container" pg_isready -U "$user" -d "$database" >/dev/null 2>&1; then + return 0 + fi + sleep 2 + done + + return 1 +} + +compose_files() { + printf -- '-f %s/docker-compose.yml -f %s/docker-compose.prod.yml ' "$SOURCE_DIR" "$SOURCE_DIR" + + if [ -f "${SOURCE_DIR}/docker-compose.custom.yml" ]; then + printf -- '-f %s/docker-compose.custom.yml ' "$SOURCE_DIR" + fi + + if [ -f "$OVERRIDE_FILE" ]; then + printf -- '-f %s ' "$OVERRIDE_FILE" + fi +} + +validate_target_major() { + case "$TARGET_MAJOR" in + ''|*[!0-9]*) + usage + fail "Target major version must be numeric. Example: $0 18" + ;; + esac + + if [ "$TARGET_MAJOR" -lt 10 ]; then + fail "Target major version must be 10 or higher." + fi +} + +mount_path_for_major() { + local major="$1" + + if [ "$major" -ge 18 ]; then + printf '%s' '/var/lib/postgresql' + else + printf '%s' '/var/lib/postgresql/data' + fi +} + +current_postgres_mount_name() { + docker inspect coolify-db --format '{{range .Mounts}}{{if or (eq .Destination "/var/lib/postgresql/data") (eq .Destination "/var/lib/postgresql")}}{{.Name}}{{end}}{{end}}' 2>/dev/null +} + +current_postgres_mount_path() { + docker inspect coolify-db --format '{{range .Mounts}}{{if or (eq .Destination "/var/lib/postgresql/data") (eq .Destination "/var/lib/postgresql")}}{{.Destination}}{{end}}{{end}}' 2>/dev/null +} + +current_postgres_image() { + docker inspect coolify-db --format '{{.Config.Image}}' 2>/dev/null +} + +write_override_file() { + local image="$1" + local volume="$2" + local mount_path="$3" + + cat > "$OVERRIDE_FILE" < "$ROLLBACK_FILE" </dev/null 2>&1 || fail "Docker is required." + docker info >/dev/null 2>&1 || fail "Docker daemon is not reachable." +} + +rollback_postgres() { + validate_common_requirements + + [ -f "$ROLLBACK_FILE" ] || fail "Missing rollback metadata file: ${ROLLBACK_FILE}" + + # shellcheck disable=SC1090 + . "$ROLLBACK_FILE" + + [ -n "${PREVIOUS_IMAGE:-}" ] || fail "Rollback metadata is missing PREVIOUS_IMAGE." + [ -n "${PREVIOUS_VOLUME:-}" ] || fail "Rollback metadata is missing PREVIOUS_VOLUME." + [ -n "${PREVIOUS_MOUNT_PATH:-}" ] || fail "Rollback metadata is missing PREVIOUS_MOUNT_PATH." + [ -n "${PREVIOUS_OVERRIDE_PRESENT:-}" ] || fail "Rollback metadata is missing PREVIOUS_OVERRIDE_PRESENT." + + log "Rolling back Coolify internal PostgreSQL." + log "Previous image: ${PREVIOUS_IMAGE}" + log "Previous volume: ${PREVIOUS_VOLUME}" + log "Previous mount path: ${PREVIOUS_MOUNT_PATH}" + + docker volume inspect "$PREVIOUS_VOLUME" >/dev/null 2>&1 || fail "Previous volume '${PREVIOUS_VOLUME}' does not exist." + + log "Stopping Coolify application container before rollback." + docker stop coolify >>"$LOGFILE" 2>&1 || true + + log "Removing current coolify-db container. Current upgraded volume is kept untouched." + docker rm -f coolify-db >>"$LOGFILE" 2>&1 || true + + if [ "$PREVIOUS_OVERRIDE_PRESENT" = "true" ]; then + log "Restoring previous PostgreSQL compose override." + write_override_file "$PREVIOUS_IMAGE" "$PREVIOUS_VOLUME" "$PREVIOUS_MOUNT_PATH" + else + log "Removing PostgreSQL compose override to restore base compose configuration." + rm -f "$OVERRIDE_FILE" + fi + + log "Starting Coolify stack with rollback database volume." + start_stack >>"$LOGFILE" 2>&1 || fail "Could not start Coolify stack after rollback. See ${LOGFILE}." + + log "Rollback completed successfully." + cat <>"$LOGFILE" 2>&1 || fail "Could not start coolify-db." + fi + + wait_for_postgres coolify-db "$DB_USERNAME" "$DB_DATABASE" || fail "Existing coolify-db is not ready." + + SERVER_VERSION_NUM=$(docker exec coolify-db psql -U "$DB_USERNAME" -d "$DB_DATABASE" -Atc 'SHOW server_version_num;' | tr -d '[:space:]') + CURRENT_MAJOR=$((SERVER_VERSION_NUM / 10000)) + PREVIOUS_VOLUME=$(current_postgres_mount_name) + PREVIOUS_MOUNT_PATH=$(current_postgres_mount_path) + PREVIOUS_IMAGE=$(current_postgres_image) + + [ -n "$PREVIOUS_VOLUME" ] || fail "Could not detect current PostgreSQL Docker volume." + [ -n "$PREVIOUS_MOUNT_PATH" ] || fail "Could not detect current PostgreSQL mount path." + [ -n "$PREVIOUS_IMAGE" ] || fail "Could not detect current PostgreSQL image." + + if [ -f "$OVERRIDE_FILE" ]; then + PREVIOUS_OVERRIDE_PRESENT=true + else + PREVIOUS_OVERRIDE_PRESENT=false + fi + + log "Current PostgreSQL major: ${CURRENT_MAJOR}" + log "Current active volume: ${PREVIOUS_VOLUME}" + log "Current image: ${PREVIOUS_IMAGE}" + log "Current mount path: ${PREVIOUS_MOUNT_PATH}" + + if [ "$CURRENT_MAJOR" -eq "$TARGET_MAJOR" ]; then + log "PostgreSQL is already on major ${TARGET_MAJOR}. Nothing to do." + exit 0 + fi + + if [ "$CURRENT_MAJOR" -gt "$TARGET_MAJOR" ]; then + fail "Downgrade from ${CURRENT_MAJOR} to ${TARGET_MAJOR} is not supported. Use '$0 rollback' to restore the previous upgrade state." + fi + + if docker volume inspect "$TARGET_VOLUME" >/dev/null 2>&1; then + fail "Target volume '${TARGET_VOLUME}' already exists. Set COOLIFY_POSTGRES_TARGET_VOLUME to a new name or remove the old failed target volume." + fi + + log "Stopping Coolify application container to prevent writes during dump." + docker stop coolify >>"$LOGFILE" 2>&1 || true + + log "Creating compressed dump at ${DUMP_FILE}." + docker exec coolify-db pg_dumpall -U "$DB_USERNAME" | gzip -c > "$DUMP_FILE" + chmod 600 "$DUMP_FILE" + + if [ ! -s "$DUMP_FILE" ]; then + fail "Dump file is empty. Aborting." + fi + + log "Creating target Docker volume '${TARGET_VOLUME}'." + docker volume create "$TARGET_VOLUME" >>"$LOGFILE" 2>&1 + + log "Pulling ${TARGET_IMAGE}." + docker pull "$TARGET_IMAGE" >>"$LOGFILE" 2>&1 + + log "Starting temporary PostgreSQL ${TARGET_MAJOR} container." + docker run -d \ + --name "$TEMP_CONTAINER" \ + --network coolify \ + -e POSTGRES_HOST_AUTH_METHOD=trust \ + -v "${TARGET_VOLUME}:${TARGET_MOUNT_PATH}" \ + "$TARGET_IMAGE" >>"$LOGFILE" 2>&1 + + wait_for_postgres "$TEMP_CONTAINER" postgres postgres || fail "Temporary PostgreSQL ${TARGET_MAJOR} container did not become ready." + + log "Restoring dump into target volume." + gunzip -c "$DUMP_FILE" | docker exec -i "$TEMP_CONTAINER" psql -U postgres -d postgres >>"$LOGFILE" 2>&1 + + log "Smoke-checking restored Coolify database." + docker exec "$TEMP_CONTAINER" psql -U "$DB_USERNAME" -d "$DB_DATABASE" -Atc 'SELECT 1;' | grep -qx '1' || fail "Restored database smoke check failed." + + log "Saving rollback metadata to ${ROLLBACK_FILE}." + write_rollback_file "$PREVIOUS_IMAGE" "$PREVIOUS_VOLUME" "$PREVIOUS_MOUNT_PATH" "$PREVIOUS_OVERRIDE_PRESENT" "$TARGET_IMAGE" "$TARGET_VOLUME" + + log "Writing Docker Compose override to ${OVERRIDE_FILE}." + write_override_file "$TARGET_IMAGE" "$TARGET_VOLUME" "$TARGET_MOUNT_PATH" + + log "Stopping temporary restore container." + docker rm -f "$TEMP_CONTAINER" >>"$LOGFILE" 2>&1 || true + + log "Stopping old coolify-db container. Previous volume '${PREVIOUS_VOLUME}' will be kept for rollback." + docker rm -f coolify-db >>"$LOGFILE" 2>&1 || true + + log "Starting Coolify stack with PostgreSQL ${TARGET_MAJOR}." + start_stack >>"$LOGFILE" 2>&1 || fail "Could not start Coolify stack with upgraded PostgreSQL. See ${LOGFILE}." + + log "Coolify internal PostgreSQL upgrade completed successfully." + print_rollback_instructions +} + +case "$COMMAND" in + rollback) + rollback_postgres + ;; + upgrade) + upgrade_postgres + ;; + -h|--help|help) + usage + ;; + *) + usage + fail "Unknown command: ${COMMAND}" + ;; +esac diff --git a/scripts/upgrade.sh b/scripts/upgrade.sh index f32db9b8d..199b531fd 100644 --- a/scripts/upgrade.sh +++ b/scripts/upgrade.sh @@ -56,6 +56,9 @@ log "Downloading docker-compose.prod.yml from ${CDN}/docker-compose.prod.yml" curl -fsSL -L $CDN/docker-compose.prod.yml -o /data/coolify/source/docker-compose.prod.yml log "Downloading .env.production from ${CDN}/.env.production" curl -fsSL -L $CDN/.env.production -o /data/coolify/source/.env.production +log "Downloading upgrade-postgres.sh from ${CDN}/upgrade-postgres.sh" +curl -fsSL -L $CDN/upgrade-postgres.sh -o /data/coolify/source/upgrade-postgres.sh +chmod +x /data/coolify/source/upgrade-postgres.sh log "Configuration files downloaded successfully" echo " Done." @@ -69,6 +72,12 @@ if [ -f /data/coolify/source/docker-compose.custom.yml ]; then log "Including custom docker-compose.yml in image extraction" fi +# Check if PostgreSQL upgrade override exists +if [ -f /data/coolify/source/docker-compose.postgres-upgrade.yml ]; then + COMPOSE_FILES="$COMPOSE_FILES -f /data/coolify/source/docker-compose.postgres-upgrade.yml" + log "Including PostgreSQL upgrade compose override in image extraction" +fi + # Get all unique images from docker compose config # LATEST_IMAGE env var is needed for image substitution in compose files IMAGES=$(LATEST_IMAGE=${LATEST_IMAGE} docker compose --env-file "$ENV_FILE" $COMPOSE_FILES config --images 2>/dev/null | sort -u) @@ -245,15 +254,18 @@ nohup bash -c " echo '============================================================' >>\"\$LOGFILE\" write_status '5' 'Starting new containers' + COMPOSE_FILES='-f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml' if [ -f /data/coolify/source/docker-compose.custom.yml ]; then log 'Using custom docker-compose.yml' - log 'Running docker compose up with custom configuration...' - docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml -f /data/coolify/source/docker-compose.custom.yml up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1 - else - log 'Using standard docker-compose configuration' - log 'Running docker compose up...' - docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1 + COMPOSE_FILES="\$COMPOSE_FILES -f /data/coolify/source/docker-compose.custom.yml" fi + if [ -f /data/coolify/source/docker-compose.postgres-upgrade.yml ]; then + log 'Using PostgreSQL upgrade compose override' + COMPOSE_FILES="\$COMPOSE_FILES -f /data/coolify/source/docker-compose.postgres-upgrade.yml" + fi + + log 'Running docker compose up...' + docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --env-file /data/coolify/source/.env \${COMPOSE_FILES} up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1 log 'Docker compose up completed' # Final log entry diff --git a/tests/Feature/SyncBunnyTest.php b/tests/Feature/SyncBunnyTest.php index 841bb5b5f..ca3091841 100644 --- a/tests/Feature/SyncBunnyTest.php +++ b/tests/Feature/SyncBunnyTest.php @@ -46,3 +46,22 @@ it('syncs nightly versions to BunnyCDN without creating a GitHub PR', function ( Http::assertSent(fn ($request) => str_starts_with($request->url(), 'https://api.bunny.net/purge') && $request['url'] === 'https://cdn.coollabs.io/coolify-nightly/versions.json'); }); + +it('syncs postgres upgrade script to BunnyCDN during full sync', function () { + Http::fake([ + 'https://cdn.coollabs.io/coolify/*' => Http::response('', 404), + 'https://storage.bunnycdn.com/*' => Http::response([], 201), + 'https://api.bunny.net/purge*' => Http::response([], 200), + ]); + + $this->artisan('sync:bunny') + ->expectsConfirmation('Are you sure you want to sync?', 'yes') + ->expectsOutputToContain('BunnyCDN sync: Complete') + ->assertExitCode(0); + + Http::assertSent(fn ($request) => $request->method() === 'PUT' + && $request->url() === 'https://storage.bunnycdn.com/coolcdn/coolify/upgrade-postgres.sh'); + + Http::assertSent(fn ($request) => str_starts_with($request->url(), 'https://api.bunny.net/purge') + && $request['url'] === 'https://cdn.coollabs.io/coolify/upgrade-postgres.sh'); +}); diff --git a/tests/Unit/UpgradePostgresScriptTest.php b/tests/Unit/UpgradePostgresScriptTest.php new file mode 100644 index 000000000..49a6b881f --- /dev/null +++ b/tests/Unit/UpgradePostgresScriptTest.php @@ -0,0 +1,93 @@ + ['pipe', 'w'], + 2 => ['pipe', 'w'], + ], + $pipes, + getcwd() + ); + + expect($process)->toBeResource(); + + $stdout = stream_get_contents($pipes[1]); + $stderr = stream_get_contents($pipes[2]); + fclose($pipes[1]); + fclose($pipes[2]); + + $exitCode = proc_close($process); + + expect($exitCode, trim($stdout."\n".$stderr))->toBe(0); +} + +it('ships postgres upgrade scripts with valid bash syntax', function () { + assertBashSyntaxIsValid('scripts/upgrade-postgres.sh'); + assertBashSyntaxIsValid('other/nightly/upgrade-postgres.sh'); +}); + +it('downloads postgres upgrade script during install and upgrade without auto-running it', function (string $path) { + $script = file_get_contents(getcwd().'/'.$path); + + expect($script) + ->toContain('upgrade-postgres.sh') + ->toContain('curl -fsSL -L $CDN/upgrade-postgres.sh -o /data/coolify/source/upgrade-postgres.sh') + ->toContain('chmod +x') + ->not->toContain('bash /data/coolify/source/upgrade-postgres.sh'); +})->with([ + 'stable install' => 'scripts/install.sh', + 'nightly install' => 'other/nightly/install.sh', + 'stable upgrade' => 'scripts/upgrade.sh', + 'nightly upgrade' => 'other/nightly/upgrade.sh', +]); + +it('keeps postgres upgrade compose override in future upgrade compose commands', function (string $path) { + $script = file_get_contents(getcwd().'/'.$path); + + expect($script) + ->toContain('docker-compose.postgres-upgrade.yml') + ->toContain('Including PostgreSQL upgrade compose override in image extraction') + ->toContain('Using PostgreSQL upgrade compose override'); +})->with([ + 'stable upgrade' => 'scripts/upgrade.sh', + 'nightly upgrade' => 'other/nightly/upgrade.sh', +]); + +it('uses postgres 18 compatible mount path in generated override and restore container', function () { + $script = file_get_contents(getcwd().'/scripts/upgrade-postgres.sh'); + + expect($script) + ->toContain("printf '%s' '/var/lib/postgresql'") + ->toContain("printf '%s' '/var/lib/postgresql/data'") + ->toContain('- coolify-db:${mount_path}') + ->toContain('-v "${TARGET_VOLUME}:${TARGET_MOUNT_PATH}"'); +}); + +it('persists rollback metadata and exposes a rollback command', function () { + $script = file_get_contents(getcwd().'/scripts/upgrade-postgres.sh'); + + expect($script) + ->toContain('ROLLBACK_FILE="${SOURCE_DIR}/postgres-upgrade-rollback.env"') + ->toContain('$0 rollback') + ->toContain('write_rollback_file') + ->toContain('PREVIOUS_VOLUME=') + ->toContain('PREVIOUS_IMAGE=') + ->toContain('PREVIOUS_MOUNT_PATH=') + ->toContain('rollback_postgres()') + ->toContain('Rollback completed successfully.'); +}); + +it('detects the active postgres volume instead of assuming coolify-db', function () { + $script = file_get_contents(getcwd().'/scripts/upgrade-postgres.sh'); + + expect($script) + ->toContain('current_postgres_mount_name()') + ->toContain('current_postgres_mount_path()') + ->toContain('current_postgres_image()') + ->toContain('Current active volume: ${PREVIOUS_VOLUME}') + ->toContain("Previous volume '") + ->toContain('will be kept for rollback'); +});