Start passthrough VM and its dependents by device identity, not address

Adding a PCIe card can renumber the buses behind it, so a hostpci entry
that names a PCI address can pass through the wrong device. These start
the VM only after finding each device by vendor/device/subsystem ID or
by its MAC-pinned interface, repoint its resource mapping if it moved,
and refuse to start on anything ambiguous. Its dependents then wait for
a port on it to answer before starting, so they find their shares up.

README.md

@@ -143,6 +143,8 @@ repository.
 | `nut/` | Reference `/etc/nut` configuration for hand-configuring a host. |
 | `pa-vm_api_stop.sh` | Graceful PA-VM firewall shutdown over the PAN-OS XML API, as `ExecStop`. |
 | `prox-maintenance-mode.sh` | Disables `onboot` autostart on every VM and container before host maintenance and restores it afterwards, saving the previous state to `/var/tmp`. `--enable`/`--disable`/`--status`, plus `--dry-run`. |
+| `start-passthrough-vm.sh` | Starts a VM with PCIe passthrough at boot only once each device has been found by ID or by its MAC-pinned interface rather than by PCI address, repointing its resource mapping if a new card has renumbered the buses. Run by `passthrough-vm-start.service`; configured in `conf/passthrough-vm.conf`. `--check` to verify without starting. |
+| `start-dependents.sh` | Starts the guests that need that VM, in order, once a port on it answers. Run by `dependents-start.service`; configured in `conf/dependents.conf`. Both scripts stand down while `prox-maintenance-mode.sh` has autostart disabled. |
 | `download-virtio-win-progress.py` | Fetches the virtio-win driver ISO. |
 | `systemd-system/` | Unit files, installed to `/etc/systemd/system`. |
 | `conf/` | Configuration, kept beside the scripts rather than in `/etc`. Committed as `.example`; the real files are not in git. |

conf/dependents.conf.example

@@ -0,0 +1,24 @@
+# Guests for start-dependents.sh, started once the passthrough VM is serving.
+#
+# Copy to conf/dependents.conf next to the scripts:
+#   cp conf/dependents.conf.example conf/dependents.conf
+#
+# The script looks for it relative to its own location, so this works both in a
+# checkout and deployed at /opt/ergosteur. Override with DEPENDENTS_CONFIG.
+#
+# Give every guest listed here onboot: 0, so that Proxmox does not start it
+# ahead of the VM it needs:
+#   qm set <vmid> --onboot 0
+#   pct set <ctid> --onboot 0
+
+# Started in this order. VM and container IDs both work.
+GUESTS=(
+    101
+    102
+)
+
+# Wait for this host:port to accept connections before starting them, for up to
+# WAIT_SECS seconds; an SMB or NFS port on the NAS, for example. Leave WAIT_FOR
+# empty to start them straight away.
+WAIT_FOR=192.0.2.10:445
+WAIT_SECS=300

conf/passthrough-vm.conf.example

@@ -0,0 +1,32 @@
+# VM and devices for start-passthrough-vm.sh.
+#
+# Copy to conf/passthrough-vm.conf next to the scripts:
+#   cp conf/passthrough-vm.conf.example conf/passthrough-vm.conf
+#
+# The script looks for it relative to its own location, so this works both in a
+# checkout and deployed at /opt/ergosteur. Override with PASSTHROUGH_VM_CONFIG.
+#
+# Give the VM onboot: 0 so that Proxmox's own autostart leaves it alone, and
+# point its hostpci entries at resource mappings rather than PCI addresses:
+#   qm set <vmid> --onboot 0
+#   qm set <vmid> --hostpci0 mapping=<mapping-id>,pcie=1
+
+VMID=100
+
+# One entry per resource mapping the VM uses: "<mapping-id> <lookup> <args>".
+#
+#   by-id <vendor:device:subvendor:subdevice>
+#       Exactly one device in the host must have these IDs, and it must already
+#       be on vfio-pci (or have no driver). Read them from sysfs:
+#       cat /sys/bus/pci/devices/<addr>/{vendor,device,subsystem_vendor,subsystem_device}
+#
+#   by-netdev <interface> <mac> <vendor:device:subvendor:subdevice>
+#       For one port of a multi-port NIC, where every port has the same IDs:
+#       the device behind <interface>, which must be pinned to <mac> by a .link
+#       file in /etc/systemd/network so that its name survives renumbering. Note
+#       that the ports of one card need not share a subsystem ID; read it from
+#       the port itself.
+DEVICES=(
+    "hba      by-id      1000:0097:1000:3090"
+    "nic-port by-netdev  nic0p1 00:11:22:33:44:55 8086:1572:8086:0000"
+)

start-dependents.sh

@@ -0,0 +1,117 @@
+#!/bin/bash
+# Start the guests that depend on the passthrough VM, once it is serving.
+#
+# Usage:
+#   ./start-dependents.sh [--check]
+#
+# Run at boot by dependents-start.service, after passthrough-vm-start.service
+# has started the VM they need (a NAS, typically). The guests to start, in
+# order, are listed in conf/dependents.conf; give each of them onboot: 0 so that
+# only this starts them. VM and container IDs both work.
+#
+# Before starting anything it waits for WAIT_FOR (host:port) to accept a TCP
+# connection, for up to WAIT_SECS seconds, because a VM that has started is not
+# yet one that is serving. If it never answers the guests are started anyway,
+# with a warning, which is no worse than not waiting at all.
+#
+# A guest that fails to start does not stop the others; the script exits
+# non-zero at the end so the unit shows as failed. Guests already running are
+# left alone. Nothing is started while prox-maintenance-mode.sh has autostart
+# disabled.
+#
+# Options:
+#   --check   Show what would be started and test WAIT_FOR once; start nothing
+#
+# The config is looked for relative to this script, so this works both in a
+# checkout and deployed at /opt/ergosteur. Override with DEPENDENTS_CONFIG.
+
+set -uo pipefail
+
+# Print the header comment block: everything from line 2 up to the first blank line.
+usage() {
+    sed -n '2,/^$/p' "$0" | sed -e 's/^#//' -e 's/^ //'
+}
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+CONFIG=${DEPENDENTS_CONFIG:-$SCRIPT_DIR/conf/dependents.conf}
+MAINT_STATE=/var/tmp/proxmox-autostart-backup.txt  # written by prox-maintenance-mode.sh --enable
+CHECK_ONLY=false
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -h|--help) usage; exit 0 ;;
+        --check)   CHECK_ONLY=true; shift ;;
+        *) echo "Error: unknown option '$1'." >&2; usage >&2; exit 1 ;;
+    esac
+done
+
+[[ -r $CONFIG ]] || { echo "Error: cannot read $CONFIG" >&2; exit 1; }
+GUESTS=()
+WAIT_FOR=""
+WAIT_SECS=300
+# shellcheck source=conf/dependents.conf.example
+source "$CONFIG"
+
+if [[ -e $MAINT_STATE ]]; then
+    echo "maintenance mode is on ($MAINT_STATE exists); not starting anything"
+    exit 0
+fi
+if [[ ${#GUESTS[@]} -eq 0 ]]; then
+    echo "no GUESTS listed in $CONFIG; nothing to do"
+    exit 0
+fi
+
+# True once host:port accepts a TCP connection.
+answers() {
+    timeout 3 bash -c "exec 3<>/dev/tcp/${1%:*}/${1##*:}" 2>/dev/null
+}
+
+if [[ -n $WAIT_FOR ]]; then
+    if $CHECK_ONLY; then
+        if answers "$WAIT_FOR"; then
+            echo "$WAIT_FOR answers"
+        else
+            echo "$WAIT_FOR is not answering right now"
+        fi
+    else
+        echo "waiting up to ${WAIT_SECS}s for $WAIT_FOR"
+        deadline=$((SECONDS + WAIT_SECS))
+        until answers "$WAIT_FOR"; do
+            if (( SECONDS >= deadline )); then
+                echo "WARNING: $WAIT_FOR still not answering after ${WAIT_SECS}s; starting the guests anyway"
+                break
+            fi
+            sleep 5
+        done
+        answers "$WAIT_FOR" && echo "$WAIT_FOR answers"
+    fi
+fi
+
+failed=0
+for id in "${GUESTS[@]}"; do
+    if [[ -e /etc/pve/qemu-server/$id.conf ]]; then
+        tool=qm conf=/etc/pve/qemu-server/$id.conf
+    elif [[ -e /etc/pve/lxc/$id.conf ]]; then
+        tool=pct conf=/etc/pve/lxc/$id.conf
+    else
+        echo "ERROR: no VM or container $id on this node" >&2
+        failed=1
+        continue
+    fi
+    if grep -q '^onboot: 1' "$conf"; then
+        echo "WARNING: $id has onboot: 1, so Proxmox starts it before the VM it needs; set onboot: 0"
+    fi
+    if [[ $($tool status "$id" 2>/dev/null) == "status: running" ]]; then
+        echo "$id is already running"
+        continue
+    fi
+    if $CHECK_ONLY; then
+        echo "would start $id ($tool)"
+    elif $tool start "$id"; then
+        echo "started $id ($tool)"
+    else
+        echo "ERROR: $tool start $id failed" >&2
+        failed=1
+    fi
+done
+exit "$failed"

start-passthrough-vm.sh

@@ -0,0 +1,206 @@
+#!/bin/bash
+# Start a VM with PCIe passthrough only after checking that its devices are where
+# its PCI resource mappings say they are.
+#
+# Usage:
+#   ./start-passthrough-vm.sh [--check]
+#
+# Run at boot by passthrough-vm-start.service, after Proxmox's own guest
+# autostart; give the VM onboot: 0 so that only this starts it. The VM and its
+# devices are set in conf/passthrough-vm.conf.
+#
+# Adding a PCIe card can renumber the buses behind it, so a hostpci entry that
+# names a PCI address can end up passing through the wrong device. Here each
+# device is found by what it is instead: by vendor/device/subsystem ID when that
+# is unique, or, for one port of a multi-port NIC, by its MAC-pinned interface.
+# If a device has moved, its resource mapping is repointed at the new address
+# before starting. If a device is missing, ambiguous, bound to a host driver or
+# sharing its IOMMU group, the VM is left stopped and the script exits non-zero,
+# so the unit shows as failed and dependents-start.service does not run.
+#
+# Nothing is started while prox-maintenance-mode.sh has autostart disabled.
+#
+# Options:
+#   --check   Verify only: report what would be done, change and start nothing
+#
+# Exit status is 0 when the VM was started, was already running, or maintenance
+# mode is on, and 1 when a check or the start failed.
+#
+# The config is looked for relative to this script, so this works both in a
+# checkout and deployed at /opt/ergosteur. Override with PASSTHROUGH_VM_CONFIG.
+
+set -uo pipefail
+
+# Print the header comment block: everything from line 2 up to the first blank line.
+usage() {
+    sed -n '2,/^$/p' "$0" | sed -e 's/^#//' -e 's/^ //'
+}
+
+SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+CONFIG=${PASSTHROUGH_VM_CONFIG:-$SCRIPT_DIR/conf/passthrough-vm.conf}
+MAINT_STATE=/var/tmp/proxmox-autostart-backup.txt  # written by prox-maintenance-mode.sh --enable
+NODE=$(hostname)
+CHECK_ONLY=false
+
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -h|--help) usage; exit 0 ;;
+        --check)   CHECK_ONLY=true; shift ;;
+        *) echo "Error: unknown option '$1'." >&2; usage >&2; exit 1 ;;
+    esac
+done
+
+[[ -r $CONFIG ]] || { echo "Error: cannot read $CONFIG" >&2; exit 1; }
+VMID=""
+DEVICES=()
+# shellcheck source=conf/passthrough-vm.conf.example
+source "$CONFIG"
+[[ -n $VMID ]] || { echo "Error: VMID is not set in $CONFIG" >&2; exit 1; }
+[[ ${#DEVICES[@]} -gt 0 ]] || { echo "Error: no DEVICES listed in $CONFIG" >&2; exit 1; }
+
+fail() {
+    echo "ABORT: $* -- VM $VMID not started" >&2
+    exit 1
+}
+
+# vendor:device:subvendor:subdevice of a PCI function, without the 0x prefixes.
+pci_ids() {
+    local d=/sys/bus/pci/devices/$1 f ids=()
+    for f in vendor device subsystem_vendor subsystem_device; do
+        ids+=("$(cut -c3- "$d/$f")")
+    done
+    (IFS=:; echo "${ids[*]}")
+}
+
+driver_of() {
+    if [[ -e /sys/bus/pci/devices/$1/driver ]]; then
+        basename "$(readlink -f "/sys/bus/pci/devices/$1/driver")"
+    else
+        echo none
+    fi
+}
+
+iommu_group() {
+    basename "$(readlink -f "/sys/bus/pci/devices/$1/iommu_group")"
+}
+
+alone_in_group() {
+    [[ $(find "/sys/bus/pci/devices/$1/iommu_group/devices/" -mindepth 1 -maxdepth 1 | wc -l) -eq 1 ]]
+}
+
+find_by_ids() {
+    local d
+    for d in /sys/bus/pci/devices/*; do
+        if [[ $(pci_ids "${d##*/}") == "$1" ]]; then
+            echo "${d##*/}"
+        fi
+    done
+}
+
+# Path of a PCI resource mapping on this node, from /etc/pve/mapping/pci.cfg.
+mapped_path() {
+    awk -v id="$1" -v node="$NODE" '
+        $0 == id { inmap = 1; next }
+        /^[^ \t]/ { inmap = 0 }
+        inmap && $1 == "map" {
+            n = split($2, kv, ","); ok = 0; path = ""
+            for (i = 1; i <= n; i++) {
+                if (kv[i] == "node=" node) ok = 1
+                if (kv[i] ~ /^path=/) path = substr(kv[i], 6)
+            }
+            if (ok) { print path; exit }
+        }' /etc/pve/mapping/pci.cfg
+}
+
+# Repoint a mapping at the device's current address if it has moved.
+# Arguments: mapping id, address, vendor:device:subvendor:subdevice, IOMMU group.
+repoint() {
+    local id=$1 addr=$2 v d sv sd current
+    current=$(mapped_path "$id")
+    [[ -n $current ]] || fail "mapping $id has no entry for node $NODE"
+    # A path without a function (0000:05:00) passes through every function of
+    # the device; keep it in that form.
+    [[ $current == *.* ]] || addr=${addr%.*}
+    if [[ $current == "$addr" ]]; then
+        echo "mapping $id ok ($addr)"
+        return
+    fi
+    echo "mapping $id points at $current but the device is at $addr"
+    if $CHECK_ONLY; then
+        echo "(check: would repoint $id to $addr)"
+        return
+    fi
+    IFS=: read -r v d sv sd <<<"$3"
+    pvesh set "/cluster/mapping/pci/$id" \
+        --map "node=$NODE,path=$addr,id=$v:$d,subsystem-id=$sv:$sd,iommugroup=$4" >/dev/null \
+        || fail "could not repoint mapping $id"
+    echo "repointed mapping $id to $addr"
+}
+
+if [[ -e $MAINT_STATE ]]; then
+    echo "maintenance mode is on ($MAINT_STATE exists); not starting VM $VMID"
+    exit 0
+fi
+
+config=$(qm config "$VMID") || fail "cannot read the config of VM $VMID"
+
+for entry in "${DEVICES[@]}"; do
+    read -r map lookup arg1 arg2 arg3 <<<"$entry"
+    case $lookup in
+        by-id)
+            want=$arg1
+            mapfile -t found < <(find_by_ids "$want")
+            [[ ${#found[@]} -eq 1 ]] || fail "$map: expected one device with IDs $want, found ${#found[@]}"
+            addr=${found[0]}
+            where="by ID"
+            # A device with a unique ID is expected to be claimed by vfio-pci at
+            # boot. One on a host driver may already be in use by the host (an
+            # HBA with its disks mounted, say), so it is not handed over.
+            case $(driver_of "$addr") in
+                vfio-pci|none) ;;
+                *) fail "$map: $addr is bound to $(driver_of "$addr"), expected vfio-pci" ;;
+            esac
+            ;;
+        by-netdev)
+            netdev=$arg1 mac=$arg2 want=$arg3
+            # At boot the port is on its host driver and visible as its pinned
+            # interface. Once the VM has run it is on vfio-pci and the interface is
+            # gone; then the mapped address is accepted as long as it is still the
+            # right kind of device. Bus numbers only change across a reboot, and
+            # after a reboot the interface is always back.
+            if [[ -e /sys/class/net/$netdev/device ]]; then
+                [[ $(cat "/sys/class/net/$netdev/address") == "$mac" ]] \
+                    || fail "$map: $netdev does not have MAC $mac"
+                addr=$(basename "$(readlink -f "/sys/class/net/$netdev/device")")
+                where="via $netdev"
+            else
+                addr=$(mapped_path "$map")
+                [[ -n $addr && -e /sys/bus/pci/devices/$addr ]] \
+                    || fail "$map: $netdev not found and the mapping has no valid path"
+                [[ $(driver_of "$addr") == vfio-pci ]] \
+                    || fail "$map: $netdev not found and $addr is not on vfio-pci"
+                where="already on vfio-pci, using the mapped address"
+            fi
+            [[ $(pci_ids "$addr") == "$want" ]] || fail "$map: $addr is $(pci_ids "$addr"), expected $want"
+            ;;
+        *)
+            fail "unknown lookup '$lookup' for $map in $CONFIG"
+            ;;
+    esac
+    alone_in_group "$addr" || fail "$map: $addr shares its IOMMU group with other devices"
+    echo "$map: $addr ($where, driver $(driver_of "$addr"), IOMMU group $(iommu_group "$addr"))"
+    grep -qE "^hostpci[0-9]+: mapping=$map(,|\$)" <<<"$config" \
+        || fail "VM $VMID has no hostpci entry using mapping $map"
+    repoint "$map" "$addr" "$want" "$(iommu_group "$addr")"
+done
+
+if [[ $(qm status "$VMID") == "status: running" ]]; then
+    echo "VM $VMID is already running"
+    exit 0
+fi
+if $CHECK_ONLY; then
+    echo "all checks passed; would start VM $VMID"
+    exit 0
+fi
+qm start "$VMID" || fail "qm start $VMID failed"
+echo "VM $VMID started"

systemd-system/dependents-start.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Start the guests that depend on the passthrough VM
+# If passthrough-vm-start.service fails this does not run, so nothing that needs
+# the VM is started without it.
+Requires=passthrough-vm-start.service
+After=passthrough-vm-start.service pve-guests.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+ExecStart=/opt/ergosteur/start-dependents.sh
+# Covers WAIT_SECS in conf/dependents.conf plus the starts themselves.
+TimeoutStartSec=900
+
+[Install]
+WantedBy=multi-user.target

systemd-system/passthrough-vm-start.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Start the passthrough VM after checking its PCIe devices
+After=pve-guests.service pve-cluster.service network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+ExecStart=/opt/ergosteur/start-passthrough-vm.sh
+TimeoutStartSec=300
+
+[Install]
+WantedBy=multi-user.target