mirror of
https://github.com/ergosteur/ergo-proxmox-scripts.git
synced 2026-10-28 13:00:37 -04:00
Start passthrough VM and its dependents by device identity, not address
Adding a PCIe card can renumber the buses behind it, so a hostpci entry that names a PCI address can pass through the wrong device. These start the VM only after finding each device by vendor/device/subsystem ID or by its MAC-pinned interface, repoint its resource mapping if it moved, and refuse to start on anything ambiguous. Its dependents then wait for a port on it to answer before starting, so they find their shares up.
This commit is contained in:
ergosteur
@@ -143,6 +143,8 @@ repository.
|
||||
| `nut/` | Reference `/etc/nut` configuration for hand-configuring a host. |
|
||||
| `pa-vm_api_stop.sh` | Graceful PA-VM firewall shutdown over the PAN-OS XML API, as `ExecStop`. |
|
||||
| `prox-maintenance-mode.sh` | Disables `onboot` autostart on every VM and container before host maintenance and restores it afterwards, saving the previous state to `/var/tmp`. `--enable`/`--disable`/`--status`, plus `--dry-run`. |
|
||||
| `start-passthrough-vm.sh` | Starts a VM with PCIe passthrough at boot only once each device has been found by ID or by its MAC-pinned interface rather than by PCI address, repointing its resource mapping if a new card has renumbered the buses. Run by `passthrough-vm-start.service`; configured in `conf/passthrough-vm.conf`. `--check` to verify without starting. |
|
||||
| `start-dependents.sh` | Starts the guests that need that VM, in order, once a port on it answers. Run by `dependents-start.service`; configured in `conf/dependents.conf`. Both scripts stand down while `prox-maintenance-mode.sh` has autostart disabled. |
|
||||
| `download-virtio-win-progress.py` | Fetches the virtio-win driver ISO. |
|
||||
| `systemd-system/` | Unit files, installed to `/etc/systemd/system`. |
|
||||
| `conf/` | Configuration, kept beside the scripts rather than in `/etc`. Committed as `.example`; the real files are not in git. |
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# Guests for start-dependents.sh, started once the passthrough VM is serving.
|
||||
#
|
||||
# Copy to conf/dependents.conf next to the scripts:
|
||||
# cp conf/dependents.conf.example conf/dependents.conf
|
||||
#
|
||||
# The script looks for it relative to its own location, so this works both in a
|
||||
# checkout and deployed at /opt/ergosteur. Override with DEPENDENTS_CONFIG.
|
||||
#
|
||||
# Give every guest listed here onboot: 0, so that Proxmox does not start it
|
||||
# ahead of the VM it needs:
|
||||
# qm set <vmid> --onboot 0
|
||||
# pct set <ctid> --onboot 0
|
||||
|
||||
# Started in this order. VM and container IDs both work.
|
||||
GUESTS=(
|
||||
101
|
||||
102
|
||||
)
|
||||
|
||||
# Wait for this host:port to accept connections before starting them, for up to
|
||||
# WAIT_SECS seconds; an SMB or NFS port on the NAS, for example. Leave WAIT_FOR
|
||||
# empty to start them straight away.
|
||||
WAIT_FOR=192.0.2.10:445
|
||||
WAIT_SECS=300
|
||||
@@ -0,0 +1,32 @@
|
||||
# VM and devices for start-passthrough-vm.sh.
|
||||
#
|
||||
# Copy to conf/passthrough-vm.conf next to the scripts:
|
||||
# cp conf/passthrough-vm.conf.example conf/passthrough-vm.conf
|
||||
#
|
||||
# The script looks for it relative to its own location, so this works both in a
|
||||
# checkout and deployed at /opt/ergosteur. Override with PASSTHROUGH_VM_CONFIG.
|
||||
#
|
||||
# Give the VM onboot: 0 so that Proxmox's own autostart leaves it alone, and
|
||||
# point its hostpci entries at resource mappings rather than PCI addresses:
|
||||
# qm set <vmid> --onboot 0
|
||||
# qm set <vmid> --hostpci0 mapping=<mapping-id>,pcie=1
|
||||
|
||||
VMID=100
|
||||
|
||||
# One entry per resource mapping the VM uses: "<mapping-id> <lookup> <args>".
|
||||
#
|
||||
# by-id <vendor:device:subvendor:subdevice>
|
||||
# Exactly one device in the host must have these IDs, and it must already
|
||||
# be on vfio-pci (or have no driver). Read them from sysfs:
|
||||
# cat /sys/bus/pci/devices/<addr>/{vendor,device,subsystem_vendor,subsystem_device}
|
||||
#
|
||||
# by-netdev <interface> <mac> <vendor:device:subvendor:subdevice>
|
||||
# For one port of a multi-port NIC, where every port has the same IDs:
|
||||
# the device behind <interface>, which must be pinned to <mac> by a .link
|
||||
# file in /etc/systemd/network so that its name survives renumbering. Note
|
||||
# that the ports of one card need not share a subsystem ID; read it from
|
||||
# the port itself.
|
||||
DEVICES=(
|
||||
"hba by-id 1000:0097:1000:3090"
|
||||
"nic-port by-netdev nic0p1 00:11:22:33:44:55 8086:1572:8086:0000"
|
||||
)
|
||||
Executable
+117
@@ -0,0 +1,117 @@
|
||||
#!/bin/bash
|
||||
# Start the guests that depend on the passthrough VM, once it is serving.
|
||||
#
|
||||
# Usage:
|
||||
# ./start-dependents.sh [--check]
|
||||
#
|
||||
# Run at boot by dependents-start.service, after passthrough-vm-start.service
|
||||
# has started the VM they need (a NAS, typically). The guests to start, in
|
||||
# order, are listed in conf/dependents.conf; give each of them onboot: 0 so that
|
||||
# only this starts them. VM and container IDs both work.
|
||||
#
|
||||
# Before starting anything it waits for WAIT_FOR (host:port) to accept a TCP
|
||||
# connection, for up to WAIT_SECS seconds, because a VM that has started is not
|
||||
# yet one that is serving. If it never answers the guests are started anyway,
|
||||
# with a warning, which is no worse than not waiting at all.
|
||||
#
|
||||
# A guest that fails to start does not stop the others; the script exits
|
||||
# non-zero at the end so the unit shows as failed. Guests already running are
|
||||
# left alone. Nothing is started while prox-maintenance-mode.sh has autostart
|
||||
# disabled.
|
||||
#
|
||||
# Options:
|
||||
# --check Show what would be started and test WAIT_FOR once; start nothing
|
||||
#
|
||||
# The config is looked for relative to this script, so this works both in a
|
||||
# checkout and deployed at /opt/ergosteur. Override with DEPENDENTS_CONFIG.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
# Print the header comment block: everything from line 2 up to the first blank line.
|
||||
usage() {
|
||||
sed -n '2,/^$/p' "$0" | sed -e 's/^#//' -e 's/^ //'
|
||||
}
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
CONFIG=${DEPENDENTS_CONFIG:-$SCRIPT_DIR/conf/dependents.conf}
|
||||
MAINT_STATE=/var/tmp/proxmox-autostart-backup.txt # written by prox-maintenance-mode.sh --enable
|
||||
CHECK_ONLY=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--help) usage; exit 0 ;;
|
||||
--check) CHECK_ONLY=true; shift ;;
|
||||
*) echo "Error: unknown option '$1'." >&2; usage >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -r $CONFIG ]] || { echo "Error: cannot read $CONFIG" >&2; exit 1; }
|
||||
GUESTS=()
|
||||
WAIT_FOR=""
|
||||
WAIT_SECS=300
|
||||
# shellcheck source=conf/dependents.conf.example
|
||||
source "$CONFIG"
|
||||
|
||||
if [[ -e $MAINT_STATE ]]; then
|
||||
echo "maintenance mode is on ($MAINT_STATE exists); not starting anything"
|
||||
exit 0
|
||||
fi
|
||||
if [[ ${#GUESTS[@]} -eq 0 ]]; then
|
||||
echo "no GUESTS listed in $CONFIG; nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# True once host:port accepts a TCP connection.
|
||||
answers() {
|
||||
timeout 3 bash -c "exec 3<>/dev/tcp/${1%:*}/${1##*:}" 2>/dev/null
|
||||
}
|
||||
|
||||
if [[ -n $WAIT_FOR ]]; then
|
||||
if $CHECK_ONLY; then
|
||||
if answers "$WAIT_FOR"; then
|
||||
echo "$WAIT_FOR answers"
|
||||
else
|
||||
echo "$WAIT_FOR is not answering right now"
|
||||
fi
|
||||
else
|
||||
echo "waiting up to ${WAIT_SECS}s for $WAIT_FOR"
|
||||
deadline=$((SECONDS + WAIT_SECS))
|
||||
until answers "$WAIT_FOR"; do
|
||||
if (( SECONDS >= deadline )); then
|
||||
echo "WARNING: $WAIT_FOR still not answering after ${WAIT_SECS}s; starting the guests anyway"
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
answers "$WAIT_FOR" && echo "$WAIT_FOR answers"
|
||||
fi
|
||||
fi
|
||||
|
||||
failed=0
|
||||
for id in "${GUESTS[@]}"; do
|
||||
if [[ -e /etc/pve/qemu-server/$id.conf ]]; then
|
||||
tool=qm conf=/etc/pve/qemu-server/$id.conf
|
||||
elif [[ -e /etc/pve/lxc/$id.conf ]]; then
|
||||
tool=pct conf=/etc/pve/lxc/$id.conf
|
||||
else
|
||||
echo "ERROR: no VM or container $id on this node" >&2
|
||||
failed=1
|
||||
continue
|
||||
fi
|
||||
if grep -q '^onboot: 1' "$conf"; then
|
||||
echo "WARNING: $id has onboot: 1, so Proxmox starts it before the VM it needs; set onboot: 0"
|
||||
fi
|
||||
if [[ $($tool status "$id" 2>/dev/null) == "status: running" ]]; then
|
||||
echo "$id is already running"
|
||||
continue
|
||||
fi
|
||||
if $CHECK_ONLY; then
|
||||
echo "would start $id ($tool)"
|
||||
elif $tool start "$id"; then
|
||||
echo "started $id ($tool)"
|
||||
else
|
||||
echo "ERROR: $tool start $id failed" >&2
|
||||
failed=1
|
||||
fi
|
||||
done
|
||||
exit "$failed"
|
||||
Executable
+206
@@ -0,0 +1,206 @@
|
||||
#!/bin/bash
|
||||
# Start a VM with PCIe passthrough only after checking that its devices are where
|
||||
# its PCI resource mappings say they are.
|
||||
#
|
||||
# Usage:
|
||||
# ./start-passthrough-vm.sh [--check]
|
||||
#
|
||||
# Run at boot by passthrough-vm-start.service, after Proxmox's own guest
|
||||
# autostart; give the VM onboot: 0 so that only this starts it. The VM and its
|
||||
# devices are set in conf/passthrough-vm.conf.
|
||||
#
|
||||
# Adding a PCIe card can renumber the buses behind it, so a hostpci entry that
|
||||
# names a PCI address can end up passing through the wrong device. Here each
|
||||
# device is found by what it is instead: by vendor/device/subsystem ID when that
|
||||
# is unique, or, for one port of a multi-port NIC, by its MAC-pinned interface.
|
||||
# If a device has moved, its resource mapping is repointed at the new address
|
||||
# before starting. If a device is missing, ambiguous, bound to a host driver or
|
||||
# sharing its IOMMU group, the VM is left stopped and the script exits non-zero,
|
||||
# so the unit shows as failed and dependents-start.service does not run.
|
||||
#
|
||||
# Nothing is started while prox-maintenance-mode.sh has autostart disabled.
|
||||
#
|
||||
# Options:
|
||||
# --check Verify only: report what would be done, change and start nothing
|
||||
#
|
||||
# Exit status is 0 when the VM was started, was already running, or maintenance
|
||||
# mode is on, and 1 when a check or the start failed.
|
||||
#
|
||||
# The config is looked for relative to this script, so this works both in a
|
||||
# checkout and deployed at /opt/ergosteur. Override with PASSTHROUGH_VM_CONFIG.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
# Print the header comment block: everything from line 2 up to the first blank line.
|
||||
usage() {
|
||||
sed -n '2,/^$/p' "$0" | sed -e 's/^#//' -e 's/^ //'
|
||||
}
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
CONFIG=${PASSTHROUGH_VM_CONFIG:-$SCRIPT_DIR/conf/passthrough-vm.conf}
|
||||
MAINT_STATE=/var/tmp/proxmox-autostart-backup.txt # written by prox-maintenance-mode.sh --enable
|
||||
NODE=$(hostname)
|
||||
CHECK_ONLY=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--help) usage; exit 0 ;;
|
||||
--check) CHECK_ONLY=true; shift ;;
|
||||
*) echo "Error: unknown option '$1'." >&2; usage >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -r $CONFIG ]] || { echo "Error: cannot read $CONFIG" >&2; exit 1; }
|
||||
VMID=""
|
||||
DEVICES=()
|
||||
# shellcheck source=conf/passthrough-vm.conf.example
|
||||
source "$CONFIG"
|
||||
[[ -n $VMID ]] || { echo "Error: VMID is not set in $CONFIG" >&2; exit 1; }
|
||||
[[ ${#DEVICES[@]} -gt 0 ]] || { echo "Error: no DEVICES listed in $CONFIG" >&2; exit 1; }
|
||||
|
||||
fail() {
|
||||
echo "ABORT: $* -- VM $VMID not started" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# vendor:device:subvendor:subdevice of a PCI function, without the 0x prefixes.
|
||||
pci_ids() {
|
||||
local d=/sys/bus/pci/devices/$1 f ids=()
|
||||
for f in vendor device subsystem_vendor subsystem_device; do
|
||||
ids+=("$(cut -c3- "$d/$f")")
|
||||
done
|
||||
(IFS=:; echo "${ids[*]}")
|
||||
}
|
||||
|
||||
driver_of() {
|
||||
if [[ -e /sys/bus/pci/devices/$1/driver ]]; then
|
||||
basename "$(readlink -f "/sys/bus/pci/devices/$1/driver")"
|
||||
else
|
||||
echo none
|
||||
fi
|
||||
}
|
||||
|
||||
iommu_group() {
|
||||
basename "$(readlink -f "/sys/bus/pci/devices/$1/iommu_group")"
|
||||
}
|
||||
|
||||
alone_in_group() {
|
||||
[[ $(find "/sys/bus/pci/devices/$1/iommu_group/devices/" -mindepth 1 -maxdepth 1 | wc -l) -eq 1 ]]
|
||||
}
|
||||
|
||||
find_by_ids() {
|
||||
local d
|
||||
for d in /sys/bus/pci/devices/*; do
|
||||
if [[ $(pci_ids "${d##*/}") == "$1" ]]; then
|
||||
echo "${d##*/}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Path of a PCI resource mapping on this node, from /etc/pve/mapping/pci.cfg.
|
||||
mapped_path() {
|
||||
awk -v id="$1" -v node="$NODE" '
|
||||
$0 == id { inmap = 1; next }
|
||||
/^[^ \t]/ { inmap = 0 }
|
||||
inmap && $1 == "map" {
|
||||
n = split($2, kv, ","); ok = 0; path = ""
|
||||
for (i = 1; i <= n; i++) {
|
||||
if (kv[i] == "node=" node) ok = 1
|
||||
if (kv[i] ~ /^path=/) path = substr(kv[i], 6)
|
||||
}
|
||||
if (ok) { print path; exit }
|
||||
}' /etc/pve/mapping/pci.cfg
|
||||
}
|
||||
|
||||
# Repoint a mapping at the device's current address if it has moved.
|
||||
# Arguments: mapping id, address, vendor:device:subvendor:subdevice, IOMMU group.
|
||||
repoint() {
|
||||
local id=$1 addr=$2 v d sv sd current
|
||||
current=$(mapped_path "$id")
|
||||
[[ -n $current ]] || fail "mapping $id has no entry for node $NODE"
|
||||
# A path without a function (0000:05:00) passes through every function of
|
||||
# the device; keep it in that form.
|
||||
[[ $current == *.* ]] || addr=${addr%.*}
|
||||
if [[ $current == "$addr" ]]; then
|
||||
echo "mapping $id ok ($addr)"
|
||||
return
|
||||
fi
|
||||
echo "mapping $id points at $current but the device is at $addr"
|
||||
if $CHECK_ONLY; then
|
||||
echo "(check: would repoint $id to $addr)"
|
||||
return
|
||||
fi
|
||||
IFS=: read -r v d sv sd <<<"$3"
|
||||
pvesh set "/cluster/mapping/pci/$id" \
|
||||
--map "node=$NODE,path=$addr,id=$v:$d,subsystem-id=$sv:$sd,iommugroup=$4" >/dev/null \
|
||||
|| fail "could not repoint mapping $id"
|
||||
echo "repointed mapping $id to $addr"
|
||||
}
|
||||
|
||||
if [[ -e $MAINT_STATE ]]; then
|
||||
echo "maintenance mode is on ($MAINT_STATE exists); not starting VM $VMID"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
config=$(qm config "$VMID") || fail "cannot read the config of VM $VMID"
|
||||
|
||||
for entry in "${DEVICES[@]}"; do
|
||||
read -r map lookup arg1 arg2 arg3 <<<"$entry"
|
||||
case $lookup in
|
||||
by-id)
|
||||
want=$arg1
|
||||
mapfile -t found < <(find_by_ids "$want")
|
||||
[[ ${#found[@]} -eq 1 ]] || fail "$map: expected one device with IDs $want, found ${#found[@]}"
|
||||
addr=${found[0]}
|
||||
where="by ID"
|
||||
# A device with a unique ID is expected to be claimed by vfio-pci at
|
||||
# boot. One on a host driver may already be in use by the host (an
|
||||
# HBA with its disks mounted, say), so it is not handed over.
|
||||
case $(driver_of "$addr") in
|
||||
vfio-pci|none) ;;
|
||||
*) fail "$map: $addr is bound to $(driver_of "$addr"), expected vfio-pci" ;;
|
||||
esac
|
||||
;;
|
||||
by-netdev)
|
||||
netdev=$arg1 mac=$arg2 want=$arg3
|
||||
# At boot the port is on its host driver and visible as its pinned
|
||||
# interface. Once the VM has run it is on vfio-pci and the interface is
|
||||
# gone; then the mapped address is accepted as long as it is still the
|
||||
# right kind of device. Bus numbers only change across a reboot, and
|
||||
# after a reboot the interface is always back.
|
||||
if [[ -e /sys/class/net/$netdev/device ]]; then
|
||||
[[ $(cat "/sys/class/net/$netdev/address") == "$mac" ]] \
|
||||
|| fail "$map: $netdev does not have MAC $mac"
|
||||
addr=$(basename "$(readlink -f "/sys/class/net/$netdev/device")")
|
||||
where="via $netdev"
|
||||
else
|
||||
addr=$(mapped_path "$map")
|
||||
[[ -n $addr && -e /sys/bus/pci/devices/$addr ]] \
|
||||
|| fail "$map: $netdev not found and the mapping has no valid path"
|
||||
[[ $(driver_of "$addr") == vfio-pci ]] \
|
||||
|| fail "$map: $netdev not found and $addr is not on vfio-pci"
|
||||
where="already on vfio-pci, using the mapped address"
|
||||
fi
|
||||
[[ $(pci_ids "$addr") == "$want" ]] || fail "$map: $addr is $(pci_ids "$addr"), expected $want"
|
||||
;;
|
||||
*)
|
||||
fail "unknown lookup '$lookup' for $map in $CONFIG"
|
||||
;;
|
||||
esac
|
||||
alone_in_group "$addr" || fail "$map: $addr shares its IOMMU group with other devices"
|
||||
echo "$map: $addr ($where, driver $(driver_of "$addr"), IOMMU group $(iommu_group "$addr"))"
|
||||
grep -qE "^hostpci[0-9]+: mapping=$map(,|\$)" <<<"$config" \
|
||||
|| fail "VM $VMID has no hostpci entry using mapping $map"
|
||||
repoint "$map" "$addr" "$want" "$(iommu_group "$addr")"
|
||||
done
|
||||
|
||||
if [[ $(qm status "$VMID") == "status: running" ]]; then
|
||||
echo "VM $VMID is already running"
|
||||
exit 0
|
||||
fi
|
||||
if $CHECK_ONLY; then
|
||||
echo "all checks passed; would start VM $VMID"
|
||||
exit 0
|
||||
fi
|
||||
qm start "$VMID" || fail "qm start $VMID failed"
|
||||
echo "VM $VMID started"
|
||||
@@ -0,0 +1,16 @@
|
||||
[Unit]
|
||||
Description=Start the guests that depend on the passthrough VM
|
||||
# If passthrough-vm-start.service fails this does not run, so nothing that needs
|
||||
# the VM is started without it.
|
||||
Requires=passthrough-vm-start.service
|
||||
After=passthrough-vm-start.service pve-guests.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=true
|
||||
ExecStart=/opt/ergosteur/start-dependents.sh
|
||||
# Covers WAIT_SECS in conf/dependents.conf plus the starts themselves.
|
||||
TimeoutStartSec=900
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,13 @@
|
||||
[Unit]
|
||||
Description=Start the passthrough VM after checking its PCIe devices
|
||||
After=pve-guests.service pve-cluster.service network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
RemainAfterExit=true
|
||||
ExecStart=/opt/ergosteur/start-passthrough-vm.sh
|
||||
TimeoutStartSec=300
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
Reference in New Issue
Block a user