From 0d55a6819dbff4d3d74ab47cc0310d1548c8bd11 Mon Sep 17 00:00:00 2001 From: ergosteur <1992147+ergosteur@users.noreply.github.com> Date: Sun, 25 Oct 2026 02:29:49 -0400 Subject: [PATCH] Start passthrough VM and its dependents by device identity, not address Adding a PCIe card can renumber the buses behind it, so a hostpci entry that names a PCI address can pass through the wrong device. These start the VM only after finding each device by vendor/device/subsystem ID or by its MAC-pinned interface, repoint its resource mapping if it moved, and refuse to start on anything ambiguous. Its dependents then wait for a port on it to answer before starting, so they find their shares up. --- README.md | 2 + conf/dependents.conf.example | 24 +++ conf/passthrough-vm.conf.example | 32 +++ start-dependents.sh | 117 +++++++++++ start-passthrough-vm.sh | 206 ++++++++++++++++++++ systemd-system/dependents-start.service | 16 ++ systemd-system/passthrough-vm-start.service | 13 ++ 7 files changed, 410 insertions(+) create mode 100644 conf/dependents.conf.example create mode 100644 conf/passthrough-vm.conf.example create mode 100755 start-dependents.sh create mode 100755 start-passthrough-vm.sh create mode 100644 systemd-system/dependents-start.service create mode 100644 systemd-system/passthrough-vm-start.service diff --git a/README.md b/README.md index 0b509c3..73e1d54 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,8 @@ repository. | `nut/` | Reference `/etc/nut` configuration for hand-configuring a host. | | `pa-vm_api_stop.sh` | Graceful PA-VM firewall shutdown over the PAN-OS XML API, as `ExecStop`. | | `prox-maintenance-mode.sh` | Disables `onboot` autostart on every VM and container before host maintenance and restores it afterwards, saving the previous state to `/var/tmp`. `--enable`/`--disable`/`--status`, plus `--dry-run`. | +| `start-passthrough-vm.sh` | Starts a VM with PCIe passthrough at boot only once each device has been found by ID or by its MAC-pinned interface rather than by PCI address, repointing its resource mapping if a new card has renumbered the buses. Run by `passthrough-vm-start.service`; configured in `conf/passthrough-vm.conf`. `--check` to verify without starting. | +| `start-dependents.sh` | Starts the guests that need that VM, in order, once a port on it answers. Run by `dependents-start.service`; configured in `conf/dependents.conf`. Both scripts stand down while `prox-maintenance-mode.sh` has autostart disabled. | | `download-virtio-win-progress.py` | Fetches the virtio-win driver ISO. | | `systemd-system/` | Unit files, installed to `/etc/systemd/system`. | | `conf/` | Configuration, kept beside the scripts rather than in `/etc`. Committed as `.example`; the real files are not in git. | diff --git a/conf/dependents.conf.example b/conf/dependents.conf.example new file mode 100644 index 0000000..390e712 --- /dev/null +++ b/conf/dependents.conf.example @@ -0,0 +1,24 @@ +# Guests for start-dependents.sh, started once the passthrough VM is serving. +# +# Copy to conf/dependents.conf next to the scripts: +# cp conf/dependents.conf.example conf/dependents.conf +# +# The script looks for it relative to its own location, so this works both in a +# checkout and deployed at /opt/ergosteur. Override with DEPENDENTS_CONFIG. +# +# Give every guest listed here onboot: 0, so that Proxmox does not start it +# ahead of the VM it needs: +# qm set --onboot 0 +# pct set --onboot 0 + +# Started in this order. VM and container IDs both work. +GUESTS=( + 101 + 102 +) + +# Wait for this host:port to accept connections before starting them, for up to +# WAIT_SECS seconds; an SMB or NFS port on the NAS, for example. Leave WAIT_FOR +# empty to start them straight away. +WAIT_FOR=192.0.2.10:445 +WAIT_SECS=300 diff --git a/conf/passthrough-vm.conf.example b/conf/passthrough-vm.conf.example new file mode 100644 index 0000000..9695e69 --- /dev/null +++ b/conf/passthrough-vm.conf.example @@ -0,0 +1,32 @@ +# VM and devices for start-passthrough-vm.sh. +# +# Copy to conf/passthrough-vm.conf next to the scripts: +# cp conf/passthrough-vm.conf.example conf/passthrough-vm.conf +# +# The script looks for it relative to its own location, so this works both in a +# checkout and deployed at /opt/ergosteur. Override with PASSTHROUGH_VM_CONFIG. +# +# Give the VM onboot: 0 so that Proxmox's own autostart leaves it alone, and +# point its hostpci entries at resource mappings rather than PCI addresses: +# qm set --onboot 0 +# qm set --hostpci0 mapping=,pcie=1 + +VMID=100 + +# One entry per resource mapping the VM uses: " ". +# +# by-id +# Exactly one device in the host must have these IDs, and it must already +# be on vfio-pci (or have no driver). Read them from sysfs: +# cat /sys/bus/pci/devices//{vendor,device,subsystem_vendor,subsystem_device} +# +# by-netdev +# For one port of a multi-port NIC, where every port has the same IDs: +# the device behind , which must be pinned to by a .link +# file in /etc/systemd/network so that its name survives renumbering. Note +# that the ports of one card need not share a subsystem ID; read it from +# the port itself. +DEVICES=( + "hba by-id 1000:0097:1000:3090" + "nic-port by-netdev nic0p1 00:11:22:33:44:55 8086:1572:8086:0000" +) diff --git a/start-dependents.sh b/start-dependents.sh new file mode 100755 index 0000000..14bd18d --- /dev/null +++ b/start-dependents.sh @@ -0,0 +1,117 @@ +#!/bin/bash +# Start the guests that depend on the passthrough VM, once it is serving. +# +# Usage: +# ./start-dependents.sh [--check] +# +# Run at boot by dependents-start.service, after passthrough-vm-start.service +# has started the VM they need (a NAS, typically). The guests to start, in +# order, are listed in conf/dependents.conf; give each of them onboot: 0 so that +# only this starts them. VM and container IDs both work. +# +# Before starting anything it waits for WAIT_FOR (host:port) to accept a TCP +# connection, for up to WAIT_SECS seconds, because a VM that has started is not +# yet one that is serving. If it never answers the guests are started anyway, +# with a warning, which is no worse than not waiting at all. +# +# A guest that fails to start does not stop the others; the script exits +# non-zero at the end so the unit shows as failed. Guests already running are +# left alone. Nothing is started while prox-maintenance-mode.sh has autostart +# disabled. +# +# Options: +# --check Show what would be started and test WAIT_FOR once; start nothing +# +# The config is looked for relative to this script, so this works both in a +# checkout and deployed at /opt/ergosteur. Override with DEPENDENTS_CONFIG. + +set -uo pipefail + +# Print the header comment block: everything from line 2 up to the first blank line. +usage() { + sed -n '2,/^$/p' "$0" | sed -e 's/^#//' -e 's/^ //' +} + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +CONFIG=${DEPENDENTS_CONFIG:-$SCRIPT_DIR/conf/dependents.conf} +MAINT_STATE=/var/tmp/proxmox-autostart-backup.txt # written by prox-maintenance-mode.sh --enable +CHECK_ONLY=false + +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) usage; exit 0 ;; + --check) CHECK_ONLY=true; shift ;; + *) echo "Error: unknown option '$1'." >&2; usage >&2; exit 1 ;; + esac +done + +[[ -r $CONFIG ]] || { echo "Error: cannot read $CONFIG" >&2; exit 1; } +GUESTS=() +WAIT_FOR="" +WAIT_SECS=300 +# shellcheck source=conf/dependents.conf.example +source "$CONFIG" + +if [[ -e $MAINT_STATE ]]; then + echo "maintenance mode is on ($MAINT_STATE exists); not starting anything" + exit 0 +fi +if [[ ${#GUESTS[@]} -eq 0 ]]; then + echo "no GUESTS listed in $CONFIG; nothing to do" + exit 0 +fi + +# True once host:port accepts a TCP connection. +answers() { + timeout 3 bash -c "exec 3<>/dev/tcp/${1%:*}/${1##*:}" 2>/dev/null +} + +if [[ -n $WAIT_FOR ]]; then + if $CHECK_ONLY; then + if answers "$WAIT_FOR"; then + echo "$WAIT_FOR answers" + else + echo "$WAIT_FOR is not answering right now" + fi + else + echo "waiting up to ${WAIT_SECS}s for $WAIT_FOR" + deadline=$((SECONDS + WAIT_SECS)) + until answers "$WAIT_FOR"; do + if (( SECONDS >= deadline )); then + echo "WARNING: $WAIT_FOR still not answering after ${WAIT_SECS}s; starting the guests anyway" + break + fi + sleep 5 + done + answers "$WAIT_FOR" && echo "$WAIT_FOR answers" + fi +fi + +failed=0 +for id in "${GUESTS[@]}"; do + if [[ -e /etc/pve/qemu-server/$id.conf ]]; then + tool=qm conf=/etc/pve/qemu-server/$id.conf + elif [[ -e /etc/pve/lxc/$id.conf ]]; then + tool=pct conf=/etc/pve/lxc/$id.conf + else + echo "ERROR: no VM or container $id on this node" >&2 + failed=1 + continue + fi + if grep -q '^onboot: 1' "$conf"; then + echo "WARNING: $id has onboot: 1, so Proxmox starts it before the VM it needs; set onboot: 0" + fi + if [[ $($tool status "$id" 2>/dev/null) == "status: running" ]]; then + echo "$id is already running" + continue + fi + if $CHECK_ONLY; then + echo "would start $id ($tool)" + elif $tool start "$id"; then + echo "started $id ($tool)" + else + echo "ERROR: $tool start $id failed" >&2 + failed=1 + fi +done +exit "$failed" diff --git a/start-passthrough-vm.sh b/start-passthrough-vm.sh new file mode 100755 index 0000000..79f5f76 --- /dev/null +++ b/start-passthrough-vm.sh @@ -0,0 +1,206 @@ +#!/bin/bash +# Start a VM with PCIe passthrough only after checking that its devices are where +# its PCI resource mappings say they are. +# +# Usage: +# ./start-passthrough-vm.sh [--check] +# +# Run at boot by passthrough-vm-start.service, after Proxmox's own guest +# autostart; give the VM onboot: 0 so that only this starts it. The VM and its +# devices are set in conf/passthrough-vm.conf. +# +# Adding a PCIe card can renumber the buses behind it, so a hostpci entry that +# names a PCI address can end up passing through the wrong device. Here each +# device is found by what it is instead: by vendor/device/subsystem ID when that +# is unique, or, for one port of a multi-port NIC, by its MAC-pinned interface. +# If a device has moved, its resource mapping is repointed at the new address +# before starting. If a device is missing, ambiguous, bound to a host driver or +# sharing its IOMMU group, the VM is left stopped and the script exits non-zero, +# so the unit shows as failed and dependents-start.service does not run. +# +# Nothing is started while prox-maintenance-mode.sh has autostart disabled. +# +# Options: +# --check Verify only: report what would be done, change and start nothing +# +# Exit status is 0 when the VM was started, was already running, or maintenance +# mode is on, and 1 when a check or the start failed. +# +# The config is looked for relative to this script, so this works both in a +# checkout and deployed at /opt/ergosteur. Override with PASSTHROUGH_VM_CONFIG. + +set -uo pipefail + +# Print the header comment block: everything from line 2 up to the first blank line. +usage() { + sed -n '2,/^$/p' "$0" | sed -e 's/^#//' -e 's/^ //' +} + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +CONFIG=${PASSTHROUGH_VM_CONFIG:-$SCRIPT_DIR/conf/passthrough-vm.conf} +MAINT_STATE=/var/tmp/proxmox-autostart-backup.txt # written by prox-maintenance-mode.sh --enable +NODE=$(hostname) +CHECK_ONLY=false + +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) usage; exit 0 ;; + --check) CHECK_ONLY=true; shift ;; + *) echo "Error: unknown option '$1'." >&2; usage >&2; exit 1 ;; + esac +done + +[[ -r $CONFIG ]] || { echo "Error: cannot read $CONFIG" >&2; exit 1; } +VMID="" +DEVICES=() +# shellcheck source=conf/passthrough-vm.conf.example +source "$CONFIG" +[[ -n $VMID ]] || { echo "Error: VMID is not set in $CONFIG" >&2; exit 1; } +[[ ${#DEVICES[@]} -gt 0 ]] || { echo "Error: no DEVICES listed in $CONFIG" >&2; exit 1; } + +fail() { + echo "ABORT: $* -- VM $VMID not started" >&2 + exit 1 +} + +# vendor:device:subvendor:subdevice of a PCI function, without the 0x prefixes. +pci_ids() { + local d=/sys/bus/pci/devices/$1 f ids=() + for f in vendor device subsystem_vendor subsystem_device; do + ids+=("$(cut -c3- "$d/$f")") + done + (IFS=:; echo "${ids[*]}") +} + +driver_of() { + if [[ -e /sys/bus/pci/devices/$1/driver ]]; then + basename "$(readlink -f "/sys/bus/pci/devices/$1/driver")" + else + echo none + fi +} + +iommu_group() { + basename "$(readlink -f "/sys/bus/pci/devices/$1/iommu_group")" +} + +alone_in_group() { + [[ $(find "/sys/bus/pci/devices/$1/iommu_group/devices/" -mindepth 1 -maxdepth 1 | wc -l) -eq 1 ]] +} + +find_by_ids() { + local d + for d in /sys/bus/pci/devices/*; do + if [[ $(pci_ids "${d##*/}") == "$1" ]]; then + echo "${d##*/}" + fi + done +} + +# Path of a PCI resource mapping on this node, from /etc/pve/mapping/pci.cfg. +mapped_path() { + awk -v id="$1" -v node="$NODE" ' + $0 == id { inmap = 1; next } + /^[^ \t]/ { inmap = 0 } + inmap && $1 == "map" { + n = split($2, kv, ","); ok = 0; path = "" + for (i = 1; i <= n; i++) { + if (kv[i] == "node=" node) ok = 1 + if (kv[i] ~ /^path=/) path = substr(kv[i], 6) + } + if (ok) { print path; exit } + }' /etc/pve/mapping/pci.cfg +} + +# Repoint a mapping at the device's current address if it has moved. +# Arguments: mapping id, address, vendor:device:subvendor:subdevice, IOMMU group. +repoint() { + local id=$1 addr=$2 v d sv sd current + current=$(mapped_path "$id") + [[ -n $current ]] || fail "mapping $id has no entry for node $NODE" + # A path without a function (0000:05:00) passes through every function of + # the device; keep it in that form. + [[ $current == *.* ]] || addr=${addr%.*} + if [[ $current == "$addr" ]]; then + echo "mapping $id ok ($addr)" + return + fi + echo "mapping $id points at $current but the device is at $addr" + if $CHECK_ONLY; then + echo "(check: would repoint $id to $addr)" + return + fi + IFS=: read -r v d sv sd <<<"$3" + pvesh set "/cluster/mapping/pci/$id" \ + --map "node=$NODE,path=$addr,id=$v:$d,subsystem-id=$sv:$sd,iommugroup=$4" >/dev/null \ + || fail "could not repoint mapping $id" + echo "repointed mapping $id to $addr" +} + +if [[ -e $MAINT_STATE ]]; then + echo "maintenance mode is on ($MAINT_STATE exists); not starting VM $VMID" + exit 0 +fi + +config=$(qm config "$VMID") || fail "cannot read the config of VM $VMID" + +for entry in "${DEVICES[@]}"; do + read -r map lookup arg1 arg2 arg3 <<<"$entry" + case $lookup in + by-id) + want=$arg1 + mapfile -t found < <(find_by_ids "$want") + [[ ${#found[@]} -eq 1 ]] || fail "$map: expected one device with IDs $want, found ${#found[@]}" + addr=${found[0]} + where="by ID" + # A device with a unique ID is expected to be claimed by vfio-pci at + # boot. One on a host driver may already be in use by the host (an + # HBA with its disks mounted, say), so it is not handed over. + case $(driver_of "$addr") in + vfio-pci|none) ;; + *) fail "$map: $addr is bound to $(driver_of "$addr"), expected vfio-pci" ;; + esac + ;; + by-netdev) + netdev=$arg1 mac=$arg2 want=$arg3 + # At boot the port is on its host driver and visible as its pinned + # interface. Once the VM has run it is on vfio-pci and the interface is + # gone; then the mapped address is accepted as long as it is still the + # right kind of device. Bus numbers only change across a reboot, and + # after a reboot the interface is always back. + if [[ -e /sys/class/net/$netdev/device ]]; then + [[ $(cat "/sys/class/net/$netdev/address") == "$mac" ]] \ + || fail "$map: $netdev does not have MAC $mac" + addr=$(basename "$(readlink -f "/sys/class/net/$netdev/device")") + where="via $netdev" + else + addr=$(mapped_path "$map") + [[ -n $addr && -e /sys/bus/pci/devices/$addr ]] \ + || fail "$map: $netdev not found and the mapping has no valid path" + [[ $(driver_of "$addr") == vfio-pci ]] \ + || fail "$map: $netdev not found and $addr is not on vfio-pci" + where="already on vfio-pci, using the mapped address" + fi + [[ $(pci_ids "$addr") == "$want" ]] || fail "$map: $addr is $(pci_ids "$addr"), expected $want" + ;; + *) + fail "unknown lookup '$lookup' for $map in $CONFIG" + ;; + esac + alone_in_group "$addr" || fail "$map: $addr shares its IOMMU group with other devices" + echo "$map: $addr ($where, driver $(driver_of "$addr"), IOMMU group $(iommu_group "$addr"))" + grep -qE "^hostpci[0-9]+: mapping=$map(,|\$)" <<<"$config" \ + || fail "VM $VMID has no hostpci entry using mapping $map" + repoint "$map" "$addr" "$want" "$(iommu_group "$addr")" +done + +if [[ $(qm status "$VMID") == "status: running" ]]; then + echo "VM $VMID is already running" + exit 0 +fi +if $CHECK_ONLY; then + echo "all checks passed; would start VM $VMID" + exit 0 +fi +qm start "$VMID" || fail "qm start $VMID failed" +echo "VM $VMID started" diff --git a/systemd-system/dependents-start.service b/systemd-system/dependents-start.service new file mode 100644 index 0000000..e290dd5 --- /dev/null +++ b/systemd-system/dependents-start.service @@ -0,0 +1,16 @@ +[Unit] +Description=Start the guests that depend on the passthrough VM +# If passthrough-vm-start.service fails this does not run, so nothing that needs +# the VM is started without it. +Requires=passthrough-vm-start.service +After=passthrough-vm-start.service pve-guests.service + +[Service] +Type=oneshot +RemainAfterExit=true +ExecStart=/opt/ergosteur/start-dependents.sh +# Covers WAIT_SECS in conf/dependents.conf plus the starts themselves. +TimeoutStartSec=900 + +[Install] +WantedBy=multi-user.target diff --git a/systemd-system/passthrough-vm-start.service b/systemd-system/passthrough-vm-start.service new file mode 100644 index 0000000..63a67f5 --- /dev/null +++ b/systemd-system/passthrough-vm-start.service @@ -0,0 +1,13 @@ +[Unit] +Description=Start the passthrough VM after checking its PCIe devices +After=pve-guests.service pve-cluster.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +RemainAfterExit=true +ExecStart=/opt/ergosteur/start-passthrough-vm.sh +TimeoutStartSec=300 + +[Install] +WantedBy=multi-user.target