fix(update-apps): keep the backup alive on a closed console, stop before restoring

A closed console killed vzdump with the session (7 runs ended "Backup failed ... creating vzdump archive"); it now runs shielded from SIGHUP. A restore after a failed update raced the background shutdown and failed with "can't overwrite running container" (3 runs); it now waits for that shutdown and for the container to stop, and says so when it does not.

tools/pve/clean-lxcs.sh

@@ -54,35 +54,35 @@ function run_lxc_clean() {
   header_info
   name=$(pct exec "$container" -- hostname)
 
-  pct exec "$container" -- sh -c '
+  pct exec "$container" -- bash -c '
     BL="\033[36m"; GN="\033[1;92m"; CL="\033[m"
     name=$(hostname)
     if [ -e /etc/alpine-release ]; then
-      printf "%b\n\n" "${BL}[Info]${GN} Cleaning $name (Alpine)${CL}"
+      echo -e "${BL}[Info]${GN} Cleaning $name (Alpine)${CL}\n"
       apk cache clean
       find /var/log -type f -delete 2>/dev/null
       find /tmp -mindepth 1 -delete 2>/dev/null
       apk update
     elif [ -e /etc/redhat-release ]; then
-      printf "%b\n\n" "${BL}[Info]${GN} Cleaning $name (CentOS)${CL}"
+      echo -e "${BL}[Info]${GN} Cleaning $name (CentOS)${CL}\n"
       yum clean all
       find /var/log -type f -delete 2>/dev/null
       find /tmp -mindepth 1 -delete 2>/dev/null
       yum update
       yum upgrade -y
     else
-      printf "%b\n\n" "${BL}[Info]${GN} Cleaning $name (Debian/Ubuntu)${CL}"
+      echo -e "${BL}[Info]${GN} Cleaning $name (Debian/Ubuntu)${CL}\n"
       find /var/cache -type f -delete 2>/dev/null
       find /var/log -type f -delete 2>/dev/null
       find /tmp -mindepth 1 -delete 2>/dev/null
       apt -y --purge autoremove
       apt -y autoclean
+      rm -rf /var/lib/apt/lists/*
       apt update
     fi
   '
 }
 
-failed_containers=()
 for container in $(pct list | awk '{if(NR>1) print $1}'); do
   excluded=0
   for ex in $excluded_containers; do
@@ -112,25 +112,17 @@ for container in $(pct list | awk '{if(NR>1) print $1}'); do
 
   if [ "$template" == "false" ] && [ "$status" == "status: stopped" ]; then
     echo -e "${BL}[Info]${GN} Starting${BL} $container ${CL} \n"
-    if ! pct start "$container"; then
-      failed_containers+=("$container (start)")
-      continue
-    fi
+    pct start "$container"
     echo -e "${BL}[Info]${GN} Waiting For${BL} $container${CL}${GN} To Start ${CL} \n"
     sleep 5
-    run_lxc_clean "$container" || failed_containers+=("$container")
+    run_lxc_clean "$container"
     echo -e "${BL}[Info]${GN} Shutting down${BL} $container ${CL} \n"
     pct shutdown "$container" &
   elif [ "$status" == "status: running" ]; then
-    run_lxc_clean "$container" || failed_containers+=("$container")
+    run_lxc_clean "$container"
   fi
 done
 
 wait
 header_info
-if [ "${#failed_containers[@]}" -gt 0 ]; then
-  echo -e "${RD}Cleaning failed in these containers:${CL}"
-  printf '%s\n' "${failed_containers[@]}"
-  exit 1
-fi
 echo -e "${GN} Finished, Selected Containers Cleaned. ${CL} \n"

tools/pve/update-apps.sh

@@ -282,7 +282,7 @@ function failure_reason() {
 function backup_container() {
   msg_info "Creating backup for container $1"
   local output
-  if output=$(vzdump $1 --compress zstd --storage $STORAGE_CHOICE -notes-template "{{guestname}} - community-scripts backup updater" 2>&1); then
+  if output=$(_cs_shield_hangup vzdump $1 --compress zstd --storage $STORAGE_CHOICE -notes-template "{{guestname}} - community-scripts backup updater" 2>&1); then
     msg_ok "Backup created"
   else
     log_write "Container $1: vzdump failed:"$'\n'"$output"
@@ -651,7 +651,18 @@ for container in $CHOICE; do
     msg_error "Update failed for container $container ($service, exit code: $exit_code) — attempting restore"
     log_write "Container $container ($service): FAILED (exit $exit_code) — attempting restore"
     msg_info "Restoring LXC $container from backup ($STORAGE_CHOICE)"
-    pct stop $container
+    wait
+    pct stop $container &>/dev/null || true
+    for _ in {1..30}; do
+      [[ "$(pct status $container 2>/dev/null)" == "status: stopped" ]] && break
+      sleep 1
+    done
+    if [[ "$(pct status $container 2>/dev/null)" != "status: stopped" ]]; then
+      msg_error "Container $container did not stop, so it cannot be restored"
+      log_result "$container" "$service" "FAILED" "Update failed (exit $exit_code) — container did not stop for the restore"
+      log_write "Container $container ($service): FAILED — container did not stop for the restore"
+      exit 235
+    fi
     LXC_STORAGE=$(pct config $container | awk -F '[:,]' '/rootfs/ {print $2}')
     BACKUP_ENTRY=$(pvesm list "$STORAGE_CHOICE" 2>/dev/null | awk -v ctid="$container" '$1 ~ "vzdump-lxc-"ctid"-" || $1 ~ "/ct/"ctid"/" {print $1}' | sort -r | head -n1)
     if [ -z "$BACKUP_ENTRY" ]; then