diff --git a/conf/ipmi-fancontrol.conf.example b/conf/ipmi-fancontrol.conf.example index e2a7c5e..b06921a 100644 --- a/conf/ipmi-fancontrol.conf.example +++ b/conf/ipmi-fancontrol.conf.example @@ -1,4 +1,4 @@ -# Credentials for the BMC user driving ipmi-fancontrol-manual / ipmi-fancontrol-auto. +# Configuration for the ipmi-fancontrol scripts and daemon. # # Copy to conf/ipmi-fancontrol.conf next to the scripts and lock it down: # cp conf/ipmi-fancontrol.conf.example conf/ipmi-fancontrol.conf @@ -7,11 +7,34 @@ # The scripts look for it relative to their own location, so this works both in a # checkout and deployed at /opt/ergosteur. Override with IPMI_FANCONTROL_CONFIG. +# How to reach the BMC. +# open - local /dev/ipmi0, needs no credentials or network (recommended +# on the host itself; requires the ipmi_si and ipmi_devintf modules) +# lanplus - over the network, using the host/user/password below +IPMI_INTERFACE=open + +# Only used when IPMI_INTERFACE=lanplus. IPMI_HOST=169.254.0.1 IPMI_USER=fancontrol - # Read by ipmitool -E, so it never appears in the process table. IPMI_PASSWORD=changeme -# Fan speed applied in manual mode, as a raw hex byte (0x15 = 21%). +# Fan duty held while the CPUs are cool, as a raw hex byte (0x15 = 21%). FAN_SPEED=0x15 + +# --- Daemon settings (ipmi-fancontrol-daemon only) --- + +# At or above this CPU temperature in degrees C, hand control back to the BMC's +# automatic curve so the fans can actually ramp. +CPU_TEMP_THRESHOLD=70 + +# Cool back down to (CPU_TEMP_THRESHOLD - CPU_TEMP_HYSTERESIS) before the fixed +# speed is re-engaged. Stops the fans oscillating around the threshold. +CPU_TEMP_HYSTERESIS=5 + +# Seconds between temperature checks. +CHECK_INTERVAL=60 + +# Dell raises the fan floor whenever it sees a third-party PCIe card. Set true to +# suppress that, which is usually what you want with aftermarket NICs or HBAs. +DISABLE_THIRD_PARTY_PCIE_COOLING_RESPONSE=false diff --git a/ipmi-fancontrol-auto b/ipmi-fancontrol-auto index 8952b87..ca08639 100755 --- a/ipmi-fancontrol-auto +++ b/ipmi-fancontrol-auto @@ -1,6 +1,7 @@ #!/bin/sh # Hand fan control back to the BMC's automatic curve. -# Paired with ipmi-fancontrol-manual via ipmi-fancontrol.service. +# +# Used ad-hoc, and as the ExecStopPost backstop for ipmi-fancontrol.service. set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) @@ -8,18 +9,31 @@ CONFIG=${IPMI_FANCONTROL_CONFIG:-$SCRIPT_DIR/conf/ipmi-fancontrol.conf} if [ ! -r "$CONFIG" ]; then echo "Error: cannot read $CONFIG" >&2 - echo "Copy conf/ipmi-fancontrol.conf.example there and fill in the BMC credentials." >&2 + echo "Copy conf/ipmi-fancontrol.conf.example there and fill it in." >&2 exit 1 fi . "$CONFIG" -: "${IPMI_HOST:?not set in $CONFIG}" -: "${IPMI_USER:?not set in $CONFIG}" -: "${IPMI_PASSWORD:?not set in $CONFIG}" +: "${IPMI_INTERFACE:=open}" -# ipmitool -E reads the password from IPMI_PASSWORD, keeping it out of argv. -export IPMI_PASSWORD +# The local "open" interface needs no credentials and no network, which matters +# here: this runs at shutdown, when networking may already be gone. +ipmi() { + if [ "$IPMI_INTERFACE" = "lanplus" ]; then + ipmitool -I lanplus -H "$IPMI_HOST" -U "$IPMI_USER" -E "$@" + else + ipmitool -I open "$@" + fi +} + +if [ "$IPMI_INTERFACE" = "lanplus" ]; then + : "${IPMI_HOST:?not set in $CONFIG}" + : "${IPMI_USER:?not set in $CONFIG}" + : "${IPMI_PASSWORD:?not set in $CONFIG}" + # ipmitool -E reads the password from the environment, keeping it out of argv. + export IPMI_PASSWORD +fi # Re-enable automatic fan control. -ipmitool -I lanplus -H "$IPMI_HOST" -U "$IPMI_USER" -E raw 0x30 0x30 0x01 0x01 +ipmi raw 0x30 0x30 0x01 0x01 diff --git a/ipmi-fancontrol-daemon b/ipmi-fancontrol-daemon new file mode 100644 index 0000000..c03f16b --- /dev/null +++ b/ipmi-fancontrol-daemon @@ -0,0 +1,122 @@ +#!/bin/bash +# Temperature-aware IPMI fan control for Dell PowerEdge / iDRAC. +# +# Holds the fans at a fixed low duty while the CPUs are cool, and hands control +# back to the BMC's automatic curve once they get hot, so there is still a +# thermal response under load. This is the native equivalent of the +# tigerblue77/dell_idrac_fan_controller container, without needing Docker. +# +# Run under ipmi-fancontrol.service. Configuration lives in +# conf/ipmi-fancontrol.conf (see conf/ipmi-fancontrol.conf.example). + +set -uo pipefail + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +CONFIG=${IPMI_FANCONTROL_CONFIG:-$SCRIPT_DIR/conf/ipmi-fancontrol.conf} + +if [[ ! -r "$CONFIG" ]]; then + echo "Error: cannot read $CONFIG" >&2 + echo "Copy conf/ipmi-fancontrol.conf.example there and fill it in." >&2 + exit 1 +fi + +# shellcheck source=/dev/null +. "$CONFIG" + +: "${IPMI_INTERFACE:=open}" +: "${FAN_SPEED:=0x15}" +: "${CPU_TEMP_THRESHOLD:=70}" +: "${CPU_TEMP_HYSTERESIS:=5}" +: "${CHECK_INTERVAL:=60}" +: "${DISABLE_THIRD_PARTY_PCIE_COOLING_RESPONSE:=false}" + +RESUME_AT=$((CPU_TEMP_THRESHOLD - CPU_TEMP_HYSTERESIS)) + +# Build the ipmitool invocation once. The local "open" interface needs no +# credentials and no network, so it is the better choice for a polling loop; +# lanplus stays available for reaching a BMC over the network. +if [[ "$IPMI_INTERFACE" == "lanplus" ]]; then + : "${IPMI_HOST:?not set in $CONFIG}" + : "${IPMI_USER:?not set in $CONFIG}" + : "${IPMI_PASSWORD:?not set in $CONFIG}" + export IPMI_PASSWORD + IPMITOOL=(ipmitool -I lanplus -H "$IPMI_HOST" -U "$IPMI_USER" -E) +else + IPMITOOL=(ipmitool -I open) +fi + +# stdout goes to the journal under systemd. +log() { echo "$*"; } + +set_manual() { + "${IPMITOOL[@]}" raw 0x30 0x30 0x01 0x00 >/dev/null && + "${IPMITOOL[@]}" raw 0x30 0x30 0x02 0xff "$FAN_SPEED" >/dev/null +} + +set_auto() { + "${IPMITOOL[@]}" raw 0x30 0x30 0x01 0x01 >/dev/null +} + +# Highest CPU temperature in degrees C, or empty if none could be read. +# CPU sensors are entity 3.x; 7.x is the system board (inlet/exhaust). +max_cpu_temp() { + "${IPMITOOL[@]}" sdr type temperature 2>/dev/null | awk -F'|' ' + $4 ~ /^[[:space:]]*3\./ { + if (match($5, /-?[0-9]+/)) { + t = substr($5, RSTART, RLENGTH) + 0 + if (!seen || t > max) max = t + seen = 1 + } + } + END { if (seen) print max } + ' +} + +MODE="" + +cleanup() { + log "Stopping: returning fan control to the BMC automatic curve." + set_auto || log "Warning: failed to restore automatic fan control." + exit 0 +} +trap cleanup TERM INT + +if [[ "$DISABLE_THIRD_PARTY_PCIE_COOLING_RESPONSE" == "true" ]]; then + if "${IPMITOOL[@]}" raw 0x30 0xce 0x00 0x16 0x05 0x00 0x00 0x00 0x05 0x00 0x01 0x00 0x00 >/dev/null 2>&1; then + log "Disabled the third-party PCIe card default cooling response." + else + log "Warning: could not disable the third-party PCIe cooling response." + fi +fi + +log "Starting: fans held at $FAN_SPEED below ${CPU_TEMP_THRESHOLD}C, automatic at or above it, re-engaging at ${RESUME_AT}C. Polling every ${CHECK_INTERVAL}s over $IPMI_INTERFACE." + +while true; do + temp=$(max_cpu_temp) + + if [[ -z "$temp" ]]; then + # Fail safe: if the temperature cannot be read, let the BMC cool the box. + if [[ "$MODE" != "auto" ]]; then + log "Cannot read CPU temperature; failing safe to automatic fan control." + set_auto && MODE="auto" + fi + elif (( temp >= CPU_TEMP_THRESHOLD )); then + if [[ "$MODE" != "auto" ]]; then + log "CPU at ${temp}C, at or above ${CPU_TEMP_THRESHOLD}C: handing control to the BMC automatic curve." + set_auto && MODE="auto" + fi + elif [[ -z "$MODE" ]] || (( temp <= RESUME_AT )); then + # The -z test makes the very first pass decide rather than drift, even + # if we start up inside the hysteresis band. + if [[ "$MODE" != "manual" ]]; then + log "CPU at ${temp}C, at or below ${RESUME_AT}C: holding fans at $FAN_SPEED." + set_manual && MODE="manual" + fi + fi + # Inside the hysteresis band we deliberately keep whatever mode we are in. + + # Backgrounded so that SIGTERM interrupts the wait instead of being deferred + # until the sleep finishes. + sleep "$CHECK_INTERVAL" & + wait $! +done diff --git a/ipmi-fancontrol-manual b/ipmi-fancontrol-manual index 8ec3a2f..02fb342 100755 --- a/ipmi-fancontrol-manual +++ b/ipmi-fancontrol-manual @@ -1,6 +1,8 @@ #!/bin/sh # Put the BMC into manual fan control and pin the fans to a fixed speed. -# Paired with ipmi-fancontrol-auto via ipmi-fancontrol.service. +# +# This is the ad-hoc override; ipmi-fancontrol.service runs the temperature-aware +# ipmi-fancontrol-daemon instead. Note that manual mode does not ramp under load. set -eu SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) @@ -8,22 +10,31 @@ CONFIG=${IPMI_FANCONTROL_CONFIG:-$SCRIPT_DIR/conf/ipmi-fancontrol.conf} if [ ! -r "$CONFIG" ]; then echo "Error: cannot read $CONFIG" >&2 - echo "Copy conf/ipmi-fancontrol.conf.example there and fill in the BMC credentials." >&2 + echo "Copy conf/ipmi-fancontrol.conf.example there and fill it in." >&2 exit 1 fi . "$CONFIG" -: "${IPMI_HOST:?not set in $CONFIG}" -: "${IPMI_USER:?not set in $CONFIG}" -: "${IPMI_PASSWORD:?not set in $CONFIG}" +: "${IPMI_INTERFACE:=open}" : "${FAN_SPEED:=0x15}" -# ipmitool -E reads the password from IPMI_PASSWORD, keeping it out of argv. -export IPMI_PASSWORD +ipmi() { + if [ "$IPMI_INTERFACE" = "lanplus" ]; then + ipmitool -I lanplus -H "$IPMI_HOST" -U "$IPMI_USER" -E "$@" + else + ipmitool -I open "$@" + fi +} -# Disable the BMC's automatic fan curve. -ipmitool -I lanplus -H "$IPMI_HOST" -U "$IPMI_USER" -E raw 0x30 0x30 0x01 0x00 +if [ "$IPMI_INTERFACE" = "lanplus" ]; then + : "${IPMI_HOST:?not set in $CONFIG}" + : "${IPMI_USER:?not set in $CONFIG}" + : "${IPMI_PASSWORD:?not set in $CONFIG}" + # ipmitool -E reads the password from the environment, keeping it out of argv. + export IPMI_PASSWORD +fi -# Apply the fixed speed to all fans. -ipmitool -I lanplus -H "$IPMI_HOST" -U "$IPMI_USER" -E raw 0x30 0x30 0x02 0xff "$FAN_SPEED" +# Disable the BMC's automatic fan curve, then apply the fixed speed to all fans. +ipmi raw 0x30 0x30 0x01 0x00 +ipmi raw 0x30 0x30 0x02 0xff "$FAN_SPEED" diff --git a/systemd-system/ipmi-fancontrol.service b/systemd-system/ipmi-fancontrol.service index 33baf75..a668ddd 100644 --- a/systemd-system/ipmi-fancontrol.service +++ b/systemd-system/ipmi-fancontrol.service @@ -1,9 +1,19 @@ [Unit] -Description=IPMI Fan Control silencing +Description=Temperature-aware IPMI fan control +Documentation=file:/opt/ergosteur/ipmi-fancontrol-daemon +# Only needed when conf sets IPMI_INTERFACE=lanplus; harmless for the local +# "open" interface. +After=network-online.target +Wants=network-online.target + [Service] -Type=oneshot -RemainAfterExit=yes -ExecStart=/opt/ergosteur/ipmi-fancontrol-manual -ExecStop=/opt/ergosteur/ipmi-fancontrol-auto +Type=simple +ExecStart=/opt/ergosteur/ipmi-fancontrol-daemon +# The daemon restores the automatic curve on SIGTERM. This is the backstop for +# the case where it is killed outright and its trap never runs. +ExecStopPost=/opt/ergosteur/ipmi-fancontrol-auto +Restart=always +RestartSec=10 + [Install] WantedBy=multi-user.target