fix(update-lxcs): keep going when one container fails

A failed patchmon-agent report, apk upgrade or pct start ended the whole run under set -e (18 runs in three days). The run now updates the other containers, lists the failed ones at the end and exits 1.

tools/pve/update-apps.sh

@@ -282,7 +282,7 @@ function failure_reason() {
 function backup_container() {
   msg_info "Creating backup for container $1"
   local output
-  if output=$(_cs_shield_hangup vzdump $1 --compress zstd --storage $STORAGE_CHOICE -notes-template "{{guestname}} - community-scripts backup updater" 2>&1); then
+  if output=$(vzdump $1 --compress zstd --storage $STORAGE_CHOICE -notes-template "{{guestname}} - community-scripts backup updater" 2>&1); then
     msg_ok "Backup created"
   else
     log_write "Container $1: vzdump failed:"$'\n'"$output"
@@ -651,18 +651,7 @@ for container in $CHOICE; do
     msg_error "Update failed for container $container ($service, exit code: $exit_code) — attempting restore"
     log_write "Container $container ($service): FAILED (exit $exit_code) — attempting restore"
     msg_info "Restoring LXC $container from backup ($STORAGE_CHOICE)"
-    wait
-    pct stop $container &>/dev/null || true
-    for _ in {1..30}; do
-      [[ "$(pct status $container 2>/dev/null)" == "status: stopped" ]] && break
-      sleep 1
-    done
-    if [[ "$(pct status $container 2>/dev/null)" != "status: stopped" ]]; then
-      msg_error "Container $container did not stop, so it cannot be restored"
-      log_result "$container" "$service" "FAILED" "Update failed (exit $exit_code) — container did not stop for the restore"
-      log_write "Container $container ($service): FAILED — container did not stop for the restore"
-      exit 235
-    fi
+    pct stop $container
     LXC_STORAGE=$(pct config $container | awk -F '[:,]' '/rootfs/ {print $2}')
     BACKUP_ENTRY=$(pvesm list "$STORAGE_CHOICE" 2>/dev/null | awk -v ctid="$container" '$1 ~ "vzdump-lxc-"ctid"-" || $1 ~ "/ct/"ctid"/" {print $1}' | sort -r | head -n1)
     if [ -z "$BACKUP_ENTRY" ]; then

tools/pve/update-lxcs.sh

@@ -84,6 +84,7 @@ function update_container() {
 }
 
 containers_needing_reboot=()
+failed_containers=()
 header_info
 for container in $(pct list | awk '{if(NR>1) print $1}'); do
   if [[ " ${excluded_containers[@]} " =~ " $container " ]]; then
@@ -101,14 +102,17 @@ for container in $(pct list | awk '{if(NR>1) print $1}'); do
     template=$(pct config $container | grep -q "template:" && echo "true" || echo "false")
     if [ "$template" == "false" ] && [ "$status" == "status: stopped" ]; then
       echo -e "${BL}[Info]${GN} Starting${BL} $container ${CL} \n"
-      pct start $container
+      if ! pct start $container; then
+        failed_containers+=("$container (start)")
+        continue
+      fi
       echo -e "${BL}[Info]${GN} Waiting For${BL} $container${CL}${GN} To Start ${CL} \n"
       sleep 5
-      update_container $container
+      update_container $container || failed_containers+=("$container")
       echo -e "${BL}[Info]${GN} Shutting down${BL} $container ${CL} \n"
       pct shutdown $container &
     elif [ "$status" == "status: running" ]; then
-      update_container $container
+      update_container $container || failed_containers+=("$container")
     fi
     if [ "$status" == "status: running" ]; then
       if pct exec "$container" -- [ -e "/var/run/reboot-required" ]; then
@@ -119,14 +123,20 @@ for container in $(pct list | awk '{if(NR>1) print $1}'); do
       # check if patchmon agent is present in container and run a report if found
       if pct exec "$container" -- [ -e "/usr/local/bin/patchmon-agent" ]; then
         echo -e "${BL}[Info]${GN} patchmon-agent found in ${BL} $container ${CL}, triggering report. \n"
-        pct exec "$container" -- "/usr/local/bin/patchmon-agent" "report"
+        pct exec "$container" -- "/usr/local/bin/patchmon-agent" "report" || echo -e "${YW}[Warn]${CL} patchmon-agent report failed in $container\n"
       fi
     fi
   fi
 done
 wait
 header_info
-echo -e "${GN}The process is complete, and the containers have been successfully updated.${CL}\n"
+if [ "${#failed_containers[@]}" -gt 0 ]; then
+  echo -e "${RD}The update failed in these containers:${CL}"
+  printf '%s\n' "${failed_containers[@]}"
+  echo ""
+else
+  echo -e "${GN}The process is complete, and the containers have been successfully updated.${CL}\n"
+fi
 if [ "${#containers_needing_reboot[@]}" -gt 0 ]; then
   echo -e "${RD}The following containers require a reboot:${CL}"
   for container_name in "${containers_needing_reboot[@]}"; do
@@ -134,3 +144,4 @@ if [ "${#containers_needing_reboot[@]}" -gt 0 ]; then
   done
 fi
 echo ""
+[ "${#failed_containers[@]}" -eq 0 ] || exit 1