Re: [PATCH v10 00/12] sched, steal_governor: Introduce preferred CPUs and steal-driven vCPU backoff

Thanks,
Yury

 From 27e548d65bbe0ec01308d3329017619e165d0488 Mon Sep 17 00:00:00 2001
From: Yury Norov <redacted>
Date: Wed, 19 Aug 2026 16:31:33 -0400
Subject: [PATCH] steal governor: add testing harness based on vng

Signed-off-by: Yury Norov <redacted>
---
  .../virtme-steal-payload.sh                   | 220 ++++++++++++
  .../steal_governor_test/virtme-steal-time.sh  | 321 ++++++++++++++++++
  2 files changed, 541 insertions(+)
  create mode 100755 drivers/virt/steal_governor_test/virtme-steal-payload.sh
  create mode 100755 drivers/virt/steal_governor_test/virtme-steal-time.sh

diff --git a/drivers/virt/steal_governor_test/virtme-steal-payload.sh b/drivers/virt/steal_governor_test/virtme-steal-payload.sh
new file mode 100755
index 000000000000..02a245a900b8
--- /dev/null
+++ b/drivers/virt/steal_governor_test/virtme-steal-payload.sh

@@ -0,0 +1,220 @@
+#!/bin/sh
+# Guest-side measurable workload for the steal governor test.
+
+set -eu
+
+case $# in
+2|3|5) ;;
+*)
+	echo "usage: ${0##*/} VM_ID DURATION_SECONDS [DRIVER [LOW_THRESHOLD HIGH_THRESHOLD]]" >&2
+	exit 2
+	;;
+esac
+
+vm_id=$1
+duration=$2
+driver=${3-}
+low_threshold=${4-}
+high_threshold=${5-}
+
+verify_parameter()
+{
+	parameter=$1
+	expected=$2
+	parameter_file=/sys/module/$driver_sysfs/parameters/$parameter
+
+	if [ ! -r "$parameter_file" ]; then
+		echo "error: driver $driver has no readable $parameter parameter" >&2
+		exit 1
+	fi
+	actual=$(cat "$parameter_file")
+	if [ "$actual" != "$expected" ]; then
+		echo "error: driver $driver $parameter is $actual, expected $expected" >&2
+		exit 1
+	fi
+}
+
+case $vm_id in
+*[!0-9]*|'')
+	echo "error: VM_ID must be a non-negative integer" >&2
+	exit 2
+	;;
+esac
+case $duration in
+*[!0-9]*|'')
+	echo "error: DURATION_SECONDS must be a positive integer" >&2
+	exit 2
+	;;
+esac
+if [ "$duration" -eq 0 ]; then
+	echo "error: DURATION_SECONDS must be greater than zero" >&2
+	exit 2
+fi
+if [ -n "$driver" ]; then
+	case $driver in
+	*[!A-Za-z0-9_.-]*)
+		echo "error: invalid driver module name: $driver" >&2
+		exit 2
+		;;
+	esac
+	if [ -n "$low_threshold" ]; then
+		case $low_threshold:$high_threshold in
+		*[!0-9:]*|:*|*:)
+			echo "error: thresholds must be non-negative integers in percent * 100" >&2
+			exit 2
+			;;
+		esac
+		if [ "$low_threshold" -ge "$high_threshold" ] ||
+		   [ "$high_threshold" -ge 10000 ]; then
+			echo "error: thresholds must satisfy 0 <= LOW < HIGH < 10000" >&2
+			exit 2
+		fi
+	fi
+	if ! command -v modprobe >/dev/null; then
+		echo "error: modprobe is required to load driver $driver" >&2
+		exit 1
+	fi
+	driver_sysfs=$(printf '%s\n' "$driver" | tr '-' '_')
+	if [ ! -d "/sys/module/$driver_sysfs" ]; then
+		if [ -n "$low_threshold" ]; then
+			modprobe "$driver" \
+				low_threshold="$low_threshold" \
+				high_threshold="$high_threshold"
+		else
+			modprobe "$driver"
+		fi
+	fi
+	if [ ! -d "/sys/module/$driver_sysfs" ]; then
+		echo "error: driver $driver has no /sys/module/$driver_sysfs entry after modprobe" >&2
+		exit 1
+	fi
+	if [ -n "$low_threshold" ]; then
+		verify_parameter low_threshold "$low_threshold"
+		verify_parameter high_threshold "$high_threshold"
+	fi
+fi
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/virtme-steal-payload.XXXXXXXX")
+before=$tmpdir/before
+after=$tmpdir/after
+pids=
+
+stop_workers()
+{
+	for pid in $pids; do
+		kill "$pid" 2>/dev/null || :
+	done
+	for pid in $pids; do
+		wait "$pid" 2>/dev/null || :
+	done
+	pids=
+}
+
+cleanup()
+{
+	stop_workers
+	rm -rf -- "$tmpdir"
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+awk '$1 ~ /^cpu[0-9]+$/ { print $1, $9 }' /proc/stat >"$before"
+start_time=$(awk '{ print $1 }' /proc/uptime)
+deadline=$(awk -v start="$start_time" -v seconds="$duration" \
+	'BEGIN { printf "%.2f", start + seconds }')
+workers=$(awk '$1 ~ /^cpu[0-9]+$/ { n++ } END { print n }' /proc/stat)
+
+i=0
+while [ "$i" -lt "$workers" ]; do
+	result=$tmpdir/work-$i
+	awk -v deadline="$deadline" -v result="$result" '
+		function uptime(    line, fields) {
+			getline line < "/proc/uptime"
+			close("/proc/uptime")
+			split(line, fields)
+			return fields[1]
+		}
+		BEGIN {
+			batch = 1000
+			units = 0
+			value = 1
+			while (uptime() < deadline) {
+				for (iteration = 0; iteration < batch; iteration++)
+					value = (value * 1103515245 + 12345) % 2147483647
+				units++
+			}
+			printf "%.0f %.0f\n", units, units * batch > result
+		}
+	' </dev/null &
+	pids="$pids $!"
+	i=$((i + 1))
+done
+
+worker_failed=0
+for pid in $pids; do
+	wait "$pid" || worker_failed=1
+done
+pids=
+if [ "$worker_failed" -ne 0 ]; then
+	echo "error: one or more workload processes failed" >&2
+	exit 1
+fi
+
+awk '$1 ~ /^cpu[0-9]+$/ { print $1, $9 }' /proc/stat >"$after"
+end_time=$(awk '{ print $1 }' /proc/uptime)
+elapsed=$(awk -v start="$start_time" -v end="$end_time" \
+	'BEGIN { printf "%.2f", end - start }')
+clk_tck=$(getconf CLK_TCK 2>/dev/null || echo 100)
+
+echo VIRTME_STEAL_REPORT_BEGIN
+driver_status=none
+[ -n "$driver" ] && driver_status=$driver:loaded
+[ -n "$low_threshold" ] && \
+	driver_status="$driver_status,thresholds=$low_threshold..$high_threshold"
+echo "VM $vm_id  kernel=$(uname -r)  vCPUs=$workers  sample=${elapsed}s  driver=$driver_status"
+echo "Workload (unpinned workers)"
+printf '%-10s %12s %16s %16s\n' \
+	WORKER WORK_UNITS ITERATIONS ITERATIONS/s
+printf '%-10s %12s %16s %16s\n' \
+	---------- ------------ ---------------- ----------------
+total_units=0
+total_iterations=0
+i=0
+while [ "$i" -lt "$workers" ]; do
+	read -r units iterations <"$tmpdir/work-$i"
+	rate=$(awk -v iterations="$iterations" -v elapsed="$elapsed" \
+		'BEGIN { printf "%.0f", iterations / elapsed }')
+	printf '%-10s %12s %16s %16s\n' \
+		"worker$i" "$units" "$iterations" "$rate"
+	total_units=$((total_units + units))
+	total_iterations=$((total_iterations + iterations))
+	i=$((i + 1))
+done
+total_rate=$(awk -v iterations="$total_iterations" -v elapsed="$elapsed" \
+	'BEGIN { printf "%.0f", iterations / elapsed }')
+throughput=$(awk -v units="$total_units" -v elapsed="$elapsed" \
+	'BEGIN { printf "%.0f", units / elapsed }')
+printf '%-10s %12s %16s %16s\n' \
+	TOTAL "$total_units" "$total_iterations" "$total_rate"
+
+echo "Steal time by guest CPU"
+printf '%-8s %12s %12s %10s\n' CPU STEAL_TICKS STEAL_s STEAL_%
+printf '%-8s %12s %12s %10s\n' -------- ------------ ------------ ----------
+average_file=$tmpdir/average-steal
+awk -v hz="$clk_tck" -v elapsed="$elapsed" -v cpus="$workers" \
+	-v average_file="$average_file" '
+	NR == FNR { before[$1] = $2; next }
+	{
+		delta = $2 - before[$1]
+		total += delta
+		printf "%-8s %12d %12.2f %10.2f\n", $1, delta,
+		       delta / hz, 100 * delta / hz / elapsed
+	}
+	END {
+		printf "%.2f\n", 100 * total / hz / elapsed / cpus > average_file
+	}
+' "$before" "$after"
+average_steal=$(cat "$average_file")
+echo "VIRTME_STEAL_SUMMARY $vm_id $throughput $average_steal"
+echo VIRTME_STEAL_REPORT_END

diff --git a/drivers/virt/steal_governor_test/virtme-steal-time.sh b/drivers/virt/steal_governor_test/virtme-steal-time.sh
new file mode 100755
index 000000000000..a41d362c7888
--- /dev/null
+++ b/drivers/virt/steal_governor_test/virtme-steal-time.sh

@@ -0,0 +1,321 @@
+#!/usr/bin/env bash
+# Build a minimal KVM guest kernel and measure per-vCPU steal time under load.
+
+set -euo pipefail
+
+script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+kernel_dir=$(CDPATH= cd -- "$script_dir/../../.." && pwd)
+vng=${VNG:-vng}
+build_dir=${BUILD_DIR:-"$kernel_dir/.virtme-steal"}
+payload=${PAYLOAD:-"$script_dir/virtme-steal-payload.sh"}
+vm_count=4
+vcpu_count=$(nproc)
+duration=20
+memory=512M
+driver=
+low_threshold=
+high_threshold=
+verbose=0
+declare -a config_items=()
+
+usage()
+{
+	cat <<EOF
+Usage: ${0##*/} [options] [O=DIR]
+
+Build a virtme-ng minimal kernel with paravirtual steal-time accounting,
+start multiple CPU-bound VMs, and report the steal time of every guest CPU.
+
+Options:
+  -n VMS       number of VMs to run (default: $vm_count)
+  -p VCPUS     vCPUs per VM (default: $vcpu_count)
+  -d SECONDS   workload duration (default: $duration)
+  -m MEMORY    memory per VM (default: $memory)
+  --configitem CONFIG[=VALUE]
+                enable or set a kernel config option (repeatable)
+  --driver MODULE
+                load and verify this module before running the payload
+  -lo VALUE     low steal threshold in percent * 100 (for example, 150 = 1.5%)
+  -hi VALUE     high steal threshold in percent * 100 (for example, 550 = 5.5%)
+  -O DIR       kernel build directory (default: $build_dir)
+  -v, --verbose show per-worker and per-CPU result tables
+  -h            show this help
+
+Environment equivalents: VNG, BUILD_DIR, and PAYLOAD. A custom payload is
+called inside each VM as: PAYLOAD VM_ID DURATION_SECONDS
+[DRIVER [LOW_THRESHOLD HIGH_THRESHOLD]].
+
+Local virtme-ng checkout example:
+  VNG=../virtme-ng/vng ${0##*/}
+
+The positional O=DIR form is equivalent to -O DIR, for example:
+  ${0##*/} -n 4 O=../build-linux-virtme-steal
+
+Config examples:
+  ${0##*/} --configitem CONFIG_SCHEDSTATS --configitem CONFIG_HZ_1000=y
+
+Threshold example:
+  ${0##*/} --driver steal_governor -lo 150 -hi 550
+EOF
+}
+
+while (($#)); do
+	case $1 in
+	-n|-p|-d|-m|-O|-lo|-hi|--configitem|--driver)
+		if (($# < 2)); then
+			echo "error: $1 requires an argument" >&2
+			exit 2
+		fi
+		case $1 in
+		-n) vm_count=$2 ;;
+		-p) vcpu_count=$2 ;;
+		-d) duration=$2 ;;
+		-m) memory=$2 ;;
+		-O) build_dir=$2 ;;
+		-lo) low_threshold=$2 ;;
+		-hi) high_threshold=$2 ;;
+		--configitem) config_items+=("$2") ;;
+		--driver) driver=$2 ;;
+		esac
+		shift 2
+		;;
+	--configitem=?*)
+		config_items+=("${1#*=}")
+		shift
+		;;
+	--driver=?*)
+		driver=${1#*=}
+		shift
+		;;
+	-v|--verbose)
+		verbose=1
+		shift
+		;;
+	-h|--help)
+		usage
+		exit 0
+		;;
+	O=?*)
+		build_dir=${1#O=}
+		shift
+		;;
+	O=|--configitem=|--driver=)
+		echo "error: ${1%%=*}= requires a non-empty argument" >&2
+		exit 2
+		;;
+	*)
+		echo "error: unexpected argument: $1" >&2
+		usage >&2
+		exit 2
+		;;
+	esac
+done
+
+require_positive_integer()
+{
+	local name=$1 value=$2
+
+	if [[ ! $value =~ ^[1-9][0-9]*$ ]]; then
+		echo "error: $name must be a positive integer (got '$value')" >&2
+		exit 2
+	fi
+}
+
+require_positive_integer "VM count" "$vm_count"
+require_positive_integer "vCPU count" "$vcpu_count"
+require_positive_integer "duration" "$duration"
+
+if [[ -n $low_threshold || -n $high_threshold ]]; then
+	if [[ -z $low_threshold || -z $high_threshold ]]; then
+		echo "error: -lo and -hi must be specified together" >&2
+		exit 2
+	fi
+	if [[ ! $low_threshold =~ ^[0-9]+$ || ! $high_threshold =~ ^[0-9]+$ ]]; then
+		echo "error: -lo and -hi must be non-negative integers in percent * 100" >&2
+		exit 2
+	fi
+	low_threshold=$((10#$low_threshold))
+	high_threshold=$((10#$high_threshold))
+	if ((low_threshold >= high_threshold)); then
+		echo "error: -lo must be less than -hi" >&2
+		exit 2
+	fi
+	if ((high_threshold >= 10000)); then
+		echo "error: -hi must be less than 10000 (100%)" >&2
+		exit 2
+	fi
+	if [[ -z $driver ]]; then
+		echo "error: -lo and -hi require --driver MODULE" >&2
+		exit 2
+	fi
+fi
+
+if [[ -n $driver && ! $driver =~ ^[A-Za-z0-9_.-]+$ ]]; then
+	echo "error: invalid driver module name: $driver" >&2
+	exit 2
+fi
+
+for index in "${!config_items[@]}"; do
+	if [[ ! ${config_items[index]} =~ ^CONFIG_[A-Z0-9_]+(=.*)?$ ]]; then
+		echo "error: invalid kernel config item: ${config_items[index]}" >&2
+		exit 2
+	fi
+	if [[ ${config_items[index]} != *=* ]]; then
+		config_items[index]="${config_items[index]}=y"
+	fi
+done
+
+if [[ ! -x $payload ]]; then
+	echo "error: guest payload is not executable: $payload" >&2
+	exit 1
+fi
+if [[ ! -r /dev/kvm || ! -w /dev/kvm ]]; then
+	echo "error: /dev/kvm is not accessible; KVM is required for steal-time accounting" >&2
+	exit 1
+fi
+
+case $build_dir in
+/*) ;;
+*) build_dir=$PWD/$build_dir ;;
+esac
+
+mkdir -p "$build_dir"
+cd "$kernel_dir"
+
+echo "==> Configuring paravirtual kernel in $build_dir"
+if [[ ! -f $build_dir/.config ]]; then
+	config_args=()
+	for config_item in "${config_items[@]}"; do
+		config_args+=(--configitem "$config_item")
+	done
+	"$vng" --kconfig "${config_args[@]}" \
+		--configitem CONFIG_HYPERVISOR_GUEST=y \
+		--configitem CONFIG_PARAVIRT=y \
+		--configitem CONFIG_KVM_GUEST=y \
+		--configitem CONFIG_PARAVIRT_TIME_ACCOUNTING=y \
+		-- "O=$build_dir"
+else
+	# Preserve an existing local minimal config and add the options needed by
+	# this test. olddefconfig resolves their dependencies for the current tree.
+	config_args=(--file "$build_dir/.config")
+	for config_item in "${config_items[@]}"; do
+		config_name=${config_item%%=*}
+		config_value=${config_item#*=}
+		config_args+=(--set-val "$config_name" "$config_value")
+	done
+	"$kernel_dir/scripts/config" "${config_args[@]}" \
+		-e HYPERVISOR_GUEST \
+		-e PARAVIRT \
+		-e KVM_GUEST \
+		-e PARAVIRT_TIME_ACCOUNTING
+	make -s O="$build_dir" olddefconfig
+fi
+
+echo "==> Building kernel"
+declare -a module_args=()
+if [[ -z $driver ]]; then
+	module_args+=(--skip-modules)
+fi
+"$vng" --build "${module_args[@]}" -- "O=$build_dir"
+
+for option in HYPERVISOR_GUEST PARAVIRT KVM_GUEST PARAVIRT_TIME_ACCOUNTING; do
+	if ! grep -qx "CONFIG_${option}=y" "$build_dir/.config"; then
+		echo "error: CONFIG_${option}=y is required but is absent after configuration" >&2
+		exit 1
+	fi
+done
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/virtme-steal.XXXXXXXX")
+declare -a vm_pids=()
+
+cleanup()
+{
+	local pid
+
+	for pid in "${vm_pids[@]}"; do
+		kill "$pid" 2>/dev/null || true
+	done
+	wait 2>/dev/null || true
+	rm -rf -- "$tmpdir"
+}
+trap cleanup EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
+
+echo "==> Starting $vm_count VMs ($vcpu_count vCPUs each)"
+for ((vm = 0; vm < vm_count; vm++)); do
+	printf -v guest_script '%q %q %q' "$payload" "$vm" "$duration"
+	if [[ -n $driver ]]; then
+		printf -v driver_arg ' %q' "$driver"
+		guest_script+=$driver_arg
+		if [[ -n $low_threshold ]]; then
+			printf -v threshold_args ' %q %q' \
+				"$low_threshold" "$high_threshold"
+			guest_script+=$threshold_args
+		fi
+	fi
+
+	log=$tmpdir/vm-$vm.log
+	"$vng" \
+		"${module_args[@]}" \
+		--name "steal-vm-$vm" \
+		--cpus "$vcpu_count" \
+		--memory "$memory" \
+		--exec "$guest_script" \
+		-- "O=$build_dir" >"$log" 2>&1 &
+	vm_pids+=("$!")
+done
+
+status=0
+total_throughput=0
+if (( ! verbose )); then
+	printf '%-8s %12s %16s\n' VM THROUGHPUT 'AVG STEAL%'
+	printf '%-8s %12s %16s\n' -------- ------------ ----------------
+fi
+for ((vm = 0; vm < vm_count; vm++)); do
+	if ! wait "${vm_pids[vm]}"; then
+		echo "error: VM $vm failed; its complete log follows" >&2
+		status=1
+		echo "--- VM $vm log ---"
+		cat "$tmpdir/vm-$vm.log"
+		continue
+	fi
+	if ! grep -qx VIRTME_STEAL_REPORT_BEGIN "$tmpdir/vm-$vm.log" ||
+	   ! grep -qx VIRTME_STEAL_REPORT_END "$tmpdir/vm-$vm.log"; then
+		echo "error: VM $vm exited without a steal-time report; its complete log follows" >&2
+		cat "$tmpdir/vm-$vm.log"
+		status=1
+		continue
+	fi
+
+	summary=$(awk '/^VIRTME_STEAL_SUMMARY / { print $2, $3, $4 }' \
+		"$tmpdir/vm-$vm.log")
+	if [[ -z $summary ]]; then
+		echo "error: VM $vm exited without a summary; its complete log follows" >&2
+		cat "$tmpdir/vm-$vm.log"
+		status=1
+		continue
+	fi
+
+	if (( verbose )); then
+		echo "--- VM $vm results ---"
+		awk '
+			/^VIRTME_STEAL_REPORT_BEGIN$/ { report = 1; next }
+			/^VIRTME_STEAL_REPORT_END$/ { report = 0 }
+			/^VIRTME_STEAL_SUMMARY / { next }
+			report
+		' "$tmpdir/vm-$vm.log"
+	else
+		read -r summary_vm summary_throughput summary_steal <<<"$summary"
+		printf '%-8s %12s %16s\n' \
+			"$summary_vm" "$summary_throughput" "$summary_steal"
+		total_throughput=$((total_throughput + summary_throughput))
+	fi
+done
+
+if (( ! verbose )); then
+	printf '%-8s %12s %16s\n' -------- ------------ ----------------
+	printf '%-8s %12s %16s\n' TOTAL "$total_throughput" -
+fi
+
+exit "$status"