mirror of
https://github.com/ergosteur/ergo-proxmox-scripts.git
synced 2026-10-28 13:00:37 -04:00
Start passthrough VM and its dependents by device identity, not address
Adding a PCIe card can renumber the buses behind it, so a hostpci entry that names a PCI address can pass through the wrong device. These start the VM only after finding each device by vendor/device/subsystem ID or by its MAC-pinned interface, repoint its resource mapping if it moved, and refuse to start on anything ambiguous. Its dependents then wait for a port on it to answer before starting, so they find their shares up.
This commit is contained in:
ergosteur
Executable
+206
@@ -0,0 +1,206 @@
|
||||
#!/bin/bash
|
||||
# Start a VM with PCIe passthrough only after checking that its devices are where
|
||||
# its PCI resource mappings say they are.
|
||||
#
|
||||
# Usage:
|
||||
# ./start-passthrough-vm.sh [--check]
|
||||
#
|
||||
# Run at boot by passthrough-vm-start.service, after Proxmox's own guest
|
||||
# autostart; give the VM onboot: 0 so that only this starts it. The VM and its
|
||||
# devices are set in conf/passthrough-vm.conf.
|
||||
#
|
||||
# Adding a PCIe card can renumber the buses behind it, so a hostpci entry that
|
||||
# names a PCI address can end up passing through the wrong device. Here each
|
||||
# device is found by what it is instead: by vendor/device/subsystem ID when that
|
||||
# is unique, or, for one port of a multi-port NIC, by its MAC-pinned interface.
|
||||
# If a device has moved, its resource mapping is repointed at the new address
|
||||
# before starting. If a device is missing, ambiguous, bound to a host driver or
|
||||
# sharing its IOMMU group, the VM is left stopped and the script exits non-zero,
|
||||
# so the unit shows as failed and dependents-start.service does not run.
|
||||
#
|
||||
# Nothing is started while prox-maintenance-mode.sh has autostart disabled.
|
||||
#
|
||||
# Options:
|
||||
# --check Verify only: report what would be done, change and start nothing
|
||||
#
|
||||
# Exit status is 0 when the VM was started, was already running, or maintenance
|
||||
# mode is on, and 1 when a check or the start failed.
|
||||
#
|
||||
# The config is looked for relative to this script, so this works both in a
|
||||
# checkout and deployed at /opt/ergosteur. Override with PASSTHROUGH_VM_CONFIG.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
# Print the header comment block: everything from line 2 up to the first blank line.
|
||||
usage() {
|
||||
sed -n '2,/^$/p' "$0" | sed -e 's/^#//' -e 's/^ //'
|
||||
}
|
||||
|
||||
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
CONFIG=${PASSTHROUGH_VM_CONFIG:-$SCRIPT_DIR/conf/passthrough-vm.conf}
|
||||
MAINT_STATE=/var/tmp/proxmox-autostart-backup.txt # written by prox-maintenance-mode.sh --enable
|
||||
NODE=$(hostname)
|
||||
CHECK_ONLY=false
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--help) usage; exit 0 ;;
|
||||
--check) CHECK_ONLY=true; shift ;;
|
||||
*) echo "Error: unknown option '$1'." >&2; usage >&2; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ -r $CONFIG ]] || { echo "Error: cannot read $CONFIG" >&2; exit 1; }
|
||||
VMID=""
|
||||
DEVICES=()
|
||||
# shellcheck source=conf/passthrough-vm.conf.example
|
||||
source "$CONFIG"
|
||||
[[ -n $VMID ]] || { echo "Error: VMID is not set in $CONFIG" >&2; exit 1; }
|
||||
[[ ${#DEVICES[@]} -gt 0 ]] || { echo "Error: no DEVICES listed in $CONFIG" >&2; exit 1; }
|
||||
|
||||
fail() {
|
||||
echo "ABORT: $* -- VM $VMID not started" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# vendor:device:subvendor:subdevice of a PCI function, without the 0x prefixes.
|
||||
pci_ids() {
|
||||
local d=/sys/bus/pci/devices/$1 f ids=()
|
||||
for f in vendor device subsystem_vendor subsystem_device; do
|
||||
ids+=("$(cut -c3- "$d/$f")")
|
||||
done
|
||||
(IFS=:; echo "${ids[*]}")
|
||||
}
|
||||
|
||||
driver_of() {
|
||||
if [[ -e /sys/bus/pci/devices/$1/driver ]]; then
|
||||
basename "$(readlink -f "/sys/bus/pci/devices/$1/driver")"
|
||||
else
|
||||
echo none
|
||||
fi
|
||||
}
|
||||
|
||||
iommu_group() {
|
||||
basename "$(readlink -f "/sys/bus/pci/devices/$1/iommu_group")"
|
||||
}
|
||||
|
||||
alone_in_group() {
|
||||
[[ $(find "/sys/bus/pci/devices/$1/iommu_group/devices/" -mindepth 1 -maxdepth 1 | wc -l) -eq 1 ]]
|
||||
}
|
||||
|
||||
find_by_ids() {
|
||||
local d
|
||||
for d in /sys/bus/pci/devices/*; do
|
||||
if [[ $(pci_ids "${d##*/}") == "$1" ]]; then
|
||||
echo "${d##*/}"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# Path of a PCI resource mapping on this node, from /etc/pve/mapping/pci.cfg.
|
||||
mapped_path() {
|
||||
awk -v id="$1" -v node="$NODE" '
|
||||
$0 == id { inmap = 1; next }
|
||||
/^[^ \t]/ { inmap = 0 }
|
||||
inmap && $1 == "map" {
|
||||
n = split($2, kv, ","); ok = 0; path = ""
|
||||
for (i = 1; i <= n; i++) {
|
||||
if (kv[i] == "node=" node) ok = 1
|
||||
if (kv[i] ~ /^path=/) path = substr(kv[i], 6)
|
||||
}
|
||||
if (ok) { print path; exit }
|
||||
}' /etc/pve/mapping/pci.cfg
|
||||
}
|
||||
|
||||
# Repoint a mapping at the device's current address if it has moved.
|
||||
# Arguments: mapping id, address, vendor:device:subvendor:subdevice, IOMMU group.
|
||||
repoint() {
|
||||
local id=$1 addr=$2 v d sv sd current
|
||||
current=$(mapped_path "$id")
|
||||
[[ -n $current ]] || fail "mapping $id has no entry for node $NODE"
|
||||
# A path without a function (0000:05:00) passes through every function of
|
||||
# the device; keep it in that form.
|
||||
[[ $current == *.* ]] || addr=${addr%.*}
|
||||
if [[ $current == "$addr" ]]; then
|
||||
echo "mapping $id ok ($addr)"
|
||||
return
|
||||
fi
|
||||
echo "mapping $id points at $current but the device is at $addr"
|
||||
if $CHECK_ONLY; then
|
||||
echo "(check: would repoint $id to $addr)"
|
||||
return
|
||||
fi
|
||||
IFS=: read -r v d sv sd <<<"$3"
|
||||
pvesh set "/cluster/mapping/pci/$id" \
|
||||
--map "node=$NODE,path=$addr,id=$v:$d,subsystem-id=$sv:$sd,iommugroup=$4" >/dev/null \
|
||||
|| fail "could not repoint mapping $id"
|
||||
echo "repointed mapping $id to $addr"
|
||||
}
|
||||
|
||||
if [[ -e $MAINT_STATE ]]; then
|
||||
echo "maintenance mode is on ($MAINT_STATE exists); not starting VM $VMID"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
config=$(qm config "$VMID") || fail "cannot read the config of VM $VMID"
|
||||
|
||||
for entry in "${DEVICES[@]}"; do
|
||||
read -r map lookup arg1 arg2 arg3 <<<"$entry"
|
||||
case $lookup in
|
||||
by-id)
|
||||
want=$arg1
|
||||
mapfile -t found < <(find_by_ids "$want")
|
||||
[[ ${#found[@]} -eq 1 ]] || fail "$map: expected one device with IDs $want, found ${#found[@]}"
|
||||
addr=${found[0]}
|
||||
where="by ID"
|
||||
# A device with a unique ID is expected to be claimed by vfio-pci at
|
||||
# boot. One on a host driver may already be in use by the host (an
|
||||
# HBA with its disks mounted, say), so it is not handed over.
|
||||
case $(driver_of "$addr") in
|
||||
vfio-pci|none) ;;
|
||||
*) fail "$map: $addr is bound to $(driver_of "$addr"), expected vfio-pci" ;;
|
||||
esac
|
||||
;;
|
||||
by-netdev)
|
||||
netdev=$arg1 mac=$arg2 want=$arg3
|
||||
# At boot the port is on its host driver and visible as its pinned
|
||||
# interface. Once the VM has run it is on vfio-pci and the interface is
|
||||
# gone; then the mapped address is accepted as long as it is still the
|
||||
# right kind of device. Bus numbers only change across a reboot, and
|
||||
# after a reboot the interface is always back.
|
||||
if [[ -e /sys/class/net/$netdev/device ]]; then
|
||||
[[ $(cat "/sys/class/net/$netdev/address") == "$mac" ]] \
|
||||
|| fail "$map: $netdev does not have MAC $mac"
|
||||
addr=$(basename "$(readlink -f "/sys/class/net/$netdev/device")")
|
||||
where="via $netdev"
|
||||
else
|
||||
addr=$(mapped_path "$map")
|
||||
[[ -n $addr && -e /sys/bus/pci/devices/$addr ]] \
|
||||
|| fail "$map: $netdev not found and the mapping has no valid path"
|
||||
[[ $(driver_of "$addr") == vfio-pci ]] \
|
||||
|| fail "$map: $netdev not found and $addr is not on vfio-pci"
|
||||
where="already on vfio-pci, using the mapped address"
|
||||
fi
|
||||
[[ $(pci_ids "$addr") == "$want" ]] || fail "$map: $addr is $(pci_ids "$addr"), expected $want"
|
||||
;;
|
||||
*)
|
||||
fail "unknown lookup '$lookup' for $map in $CONFIG"
|
||||
;;
|
||||
esac
|
||||
alone_in_group "$addr" || fail "$map: $addr shares its IOMMU group with other devices"
|
||||
echo "$map: $addr ($where, driver $(driver_of "$addr"), IOMMU group $(iommu_group "$addr"))"
|
||||
grep -qE "^hostpci[0-9]+: mapping=$map(,|\$)" <<<"$config" \
|
||||
|| fail "VM $VMID has no hostpci entry using mapping $map"
|
||||
repoint "$map" "$addr" "$want" "$(iommu_group "$addr")"
|
||||
done
|
||||
|
||||
if [[ $(qm status "$VMID") == "status: running" ]]; then
|
||||
echo "VM $VMID is already running"
|
||||
exit 0
|
||||
fi
|
||||
if $CHECK_ONLY; then
|
||||
echo "all checks passed; would start VM $VMID"
|
||||
exit 0
|
||||
fi
|
||||
qm start "$VMID" || fail "qm start $VMID failed"
|
||||
echo "VM $VMID started"
|
||||
Reference in New Issue
Block a user