From 6918c9bd298ec079bcad32968c8d81079c3f5343 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 10:41:36 -0500 Subject: [PATCH 1/6] chore(vm): remove unused Packer template and image builder The harness boots the Alpine live ISO directly, so the Packer template, build-vm-image.sh and the vm-entrypoint.sh they installed are dead code. Co-Authored-By: Claude Opus 5.5 --- tests/vm/packer/alpine.pkr.hcl | 184 ------------------------ tests/vm/scripts/build-vm-image.sh | 217 ----------------------------- tests/vm/scripts/vm-entrypoint.sh | 122 ---------------- 3 files changed, 523 deletions(-) delete mode 100644 tests/vm/packer/alpine.pkr.hcl delete mode 100755 tests/vm/scripts/build-vm-image.sh delete mode 100755 tests/vm/scripts/vm-entrypoint.sh diff --git a/tests/vm/packer/alpine.pkr.hcl b/tests/vm/packer/alpine.pkr.hcl deleted file mode 100644 index 433f446f..00000000 --- a/tests/vm/packer/alpine.pkr.hcl +++ /dev/null @@ -1,184 +0,0 @@ -# GoClusterFS Alpine VM Image -# Packer template using pre-built Alpine cloud image (more reliable than ISO install) -# -# Usage: -# export GOCLUSTERFS_BINARY=/path/to/goclusterfs-mount -# export SSH_PUBLIC_KEY="ssh-ed25519 AAAA..." -# export OUTPUT_DIR=/path/to/output -# packer build alpine.pkr.hcl - -packer { - required_plugins { - qemu = { - version = ">= 1.0.0" - source = "github.com/hashicorp/qemu" - } - } -} - -# Variables -variable "alpine_version" { - type = string - default = "3.19" -} - -variable "output_directory" { - type = string - default = env("OUTPUT_DIR") -} - -variable "goclusterfs_binary" { - type = string - default = env("GOCLUSTERFS_BINARY") -} - -variable "ssh_public_key" { - type = string - default = env("SSH_PUBLIC_KEY") -} - -variable "disk_size" { - type = string - default = "1G" -} - -variable "memory" { - type = string - default = "512" -} - -# Local variables -locals { - # Alpine provides pre-built cloud images in QCOW2 format - cloud_image_url = "https://dl-cdn.alpinelinux.org/alpine/v${var.alpine_version}/releases/x86_64/alpine-virt-${var.alpine_version}.0-x86_64.iso" - output_dir = var.output_directory != "" ? var.output_directory : "${path.root}/../runtime/packer-output" -} - -# QEMU builder - use a simpler approach with direct console access -source "qemu" "alpine" { - iso_url = local.cloud_image_url - iso_checksum = "file:https://dl-cdn.alpinelinux.org/alpine/v${var.alpine_version}/releases/x86_64/alpine-virt-${var.alpine_version}.0-x86_64.iso.sha256" - output_directory = local.output_dir - vm_name = "goclusterfs-alpine.qcow2" - - # VM configuration - disk_size = var.disk_size - memory = var.memory - cpus = 1 - accelerator = "kvm" - format = "qcow2" - disk_interface = "virtio" - net_device = "virtio-net" - - # Boot settings - simplified for reliability - boot_wait = "30s" - boot_command = [ - # Login as root (no password on live system) - "root", - - # Set up basic networking - "setup-interfaces -a", - "ifup eth0", - - # Set root password for SSH - "echo 'root:testpass123' | chpasswd", - - # Enable and configure SSH - "apk add openssh", - "sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config", - "sed -i 's/#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config", - "rc-service sshd start", - "rc-update add sshd", - - # Set up disk for installation - "setup-disk -m sys -s 0 /dev/vda", - "y", - - # Reboot into installed system - "reboot", - - # Login after reboot - "root", - "testpass123", - - # Re-enable SSH after reboot - "rc-service sshd start", - ] - - # SSH configuration for provisioning - ssh_username = "root" - ssh_password = "testpass123" - ssh_timeout = "15m" - shutdown_command = "poweroff" - - # Headless mode - headless = true - - # QEMU display - display = "none" -} - -# Build -build { - sources = ["source.qemu.alpine"] - - # Install required packages - provisioner "shell" { - inline = [ - "apk update", - "apk add fuse3 fuse3-dev util-linux procps", - "modprobe fuse || true", - "echo 'fuse' >> /etc/modules", - ] - } - - # Set up SSH key authentication - provisioner "shell" { - inline = [ - "mkdir -p /root/.ssh", - "chmod 700 /root/.ssh", - "echo '${var.ssh_public_key}' > /root/.ssh/authorized_keys", - "chmod 600 /root/.ssh/authorized_keys", - ] - } - - # Copy GoClusterFS binary - provisioner "file" { - source = var.goclusterfs_binary - destination = "/usr/local/bin/goclusterfs-mount" - } - - provisioner "shell" { - inline = [ - "chmod +x /usr/local/bin/goclusterfs-mount", - ] - } - - # Copy entrypoint script - provisioner "file" { - source = "${path.root}/../scripts/vm-entrypoint.sh" - destination = "/usr/local/bin/vm-entrypoint.sh" - } - - provisioner "shell" { - inline = [ - "chmod +x /usr/local/bin/vm-entrypoint.sh", - ] - } - - # Set up auto-start of GoClusterFS - provisioner "shell" { - inline = [ - # Create init script for GoClusterFS - "cat > /etc/init.d/goclusterfs << 'INITEOF'\n#!/sbin/openrc-run\n\ndepend() {\n need localmount\n after modules\n}\n\nstart() {\n ebegin \"Starting GoClusterFS\"\n /usr/local/bin/vm-entrypoint.sh &\n eend $?\n}\n\nstop() {\n ebegin \"Stopping GoClusterFS\"\n fusermount -u /mnt/goclusterfs 2>/dev/null || true\n eend $?\n}\nINITEOF", - "chmod +x /etc/init.d/goclusterfs", - "rc-update add goclusterfs default", - - # Create mount point - "mkdir -p /mnt/goclusterfs", - - # Clean up - "rm -f /root/.ash_history", - ] - } -} diff --git a/tests/vm/scripts/build-vm-image.sh b/tests/vm/scripts/build-vm-image.sh deleted file mode 100755 index b66249be..00000000 --- a/tests/vm/scripts/build-vm-image.sh +++ /dev/null @@ -1,217 +0,0 @@ -#!/bin/bash -# GoClusterFS VM Image Builder -# Creates an Alpine-based VM image for testing without Packer -# -# This script: -# 1. Downloads Alpine minirootfs -# 2. Creates a QCOW2 disk image -# 3. Installs Alpine to the image -# 4. Adds GoClusterFS and dependencies - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" -RUNTIME_DIR="${SCRIPT_DIR}/../runtime" -OUTPUT_DIR="${RUNTIME_DIR}/packer-output" - -# Alpine settings -ALPINE_VERSION="3.19" -ALPINE_MIRROR="https://dl-cdn.alpinelinux.org/alpine" -MINIROOTFS_URL="${ALPINE_MIRROR}/v${ALPINE_VERSION}/releases/x86_64/alpine-minirootfs-${ALPINE_VERSION}.0-x86_64.tar.gz" - -# Image settings -IMAGE_NAME="goclusterfs-alpine.qcow2" -IMAGE_SIZE="512M" -MOUNT_POINT="/tmp/goclusterfs-vm-build" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -BLUE='\033[0;34m' -NC='\033[0m' - -log_info() { echo -e "${BLUE}[INFO]${NC} $*"; } -log_success() { echo -e "${GREEN}[OK]${NC} $*"; } -log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } - -cleanup() { - log_info "Cleaning up..." - sudo umount "${MOUNT_POINT}" 2>/dev/null || true - sudo losetup -D 2>/dev/null || true - rm -rf "${MOUNT_POINT}" -} - -trap cleanup EXIT - -# Check dependencies -check_deps() { - local missing=() - for cmd in qemu-img wget tar; do - if ! command -v "$cmd" &>/dev/null; then - missing+=("$cmd") - fi - done - - if [[ ${#missing[@]} -gt 0 ]]; then - log_error "Missing dependencies: ${missing[*]}" - exit 1 - fi - - if [[ ! -f "${PROJECT_ROOT}/goclusterfs-mount" ]]; then - log_error "goclusterfs-mount binary not found. Run 'make build-mount' first." - exit 1 - fi -} - -# Download Alpine minirootfs -download_rootfs() { - local tarball="${RUNTIME_DIR}/alpine-minirootfs.tar.gz" - - if [[ -f "$tarball" ]]; then - log_info "Using cached minirootfs" - return - fi - - log_info "Downloading Alpine minirootfs..." - wget -q --show-progress -O "$tarball" "$MINIROOTFS_URL" - log_success "Downloaded minirootfs" -} - -# Create and populate the image -build_image() { - mkdir -p "${OUTPUT_DIR}" - local image_path="${OUTPUT_DIR}/${IMAGE_NAME}" - local raw_image="${OUTPUT_DIR}/temp.raw" - - if [[ -f "$image_path" ]]; then - log_info "Removing existing image..." - rm -f "$image_path" - fi - - log_info "Creating disk image..." - - # Create raw image (easier to work with) - dd if=/dev/zero of="$raw_image" bs=1M count=512 status=progress - - # Create partition and filesystem - log_info "Creating filesystem..." - mkfs.ext4 -F "$raw_image" - - # Mount and populate - mkdir -p "${MOUNT_POINT}" - sudo mount -o loop "$raw_image" "${MOUNT_POINT}" - - log_info "Extracting Alpine rootfs..." - sudo tar -xzf "${RUNTIME_DIR}/alpine-minirootfs.tar.gz" -C "${MOUNT_POINT}" - - # Configure Alpine - log_info "Configuring Alpine..." - - # Set up resolv.conf - echo "nameserver 8.8.8.8" | sudo tee "${MOUNT_POINT}/etc/resolv.conf" > /dev/null - - # Set up repositories - cat < /dev/null -${ALPINE_MIRROR}/v${ALPINE_VERSION}/main -${ALPINE_MIRROR}/v${ALPINE_VERSION}/community -EOF - - # Install packages using chroot - log_info "Installing packages..." - sudo chroot "${MOUNT_POINT}" /bin/sh -c " - apk update - apk add --no-cache openssh fuse3 util-linux procps openrc e2fsprogs - - # Configure SSH - ssh-keygen -A - sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config - sed -i 's/#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config - echo 'root:testpass123' | chpasswd - - # Enable services - rc-update add sshd default - rc-update add devfs sysinit - rc-update add dmesg sysinit - rc-update add mdev sysinit - - # Create mount point - mkdir -p /mnt/goclusterfs - " - - # Copy GoClusterFS binary - log_info "Installing GoClusterFS..." - sudo cp "${PROJECT_ROOT}/goclusterfs-mount" "${MOUNT_POINT}/usr/local/bin/" - sudo chmod +x "${MOUNT_POINT}/usr/local/bin/goclusterfs-mount" - - # Copy entrypoint script - sudo cp "${SCRIPT_DIR}/vm-entrypoint.sh" "${MOUNT_POINT}/usr/local/bin/" - sudo chmod +x "${MOUNT_POINT}/usr/local/bin/vm-entrypoint.sh" - - # Set up SSH authorized keys if provided - if [[ -f "${RUNTIME_DIR}/test_key.pub" ]]; then - sudo mkdir -p "${MOUNT_POINT}/root/.ssh" - sudo cp "${RUNTIME_DIR}/test_key.pub" "${MOUNT_POINT}/root/.ssh/authorized_keys" - sudo chmod 700 "${MOUNT_POINT}/root/.ssh" - sudo chmod 600 "${MOUNT_POINT}/root/.ssh/authorized_keys" - fi - - # Create GoClusterFS init script - cat <<'INITEOF' | sudo tee "${MOUNT_POINT}/etc/init.d/goclusterfs" > /dev/null -#!/sbin/openrc-run - -depend() { - need localmount - after modules -} - -start() { - ebegin "Starting GoClusterFS" - /usr/local/bin/vm-entrypoint.sh & - eend $? -} - -stop() { - ebegin "Stopping GoClusterFS" - fusermount -u /mnt/goclusterfs 2>/dev/null || true - eend $? -} -INITEOF - sudo chmod +x "${MOUNT_POINT}/etc/init.d/goclusterfs" - sudo chroot "${MOUNT_POINT}" rc-update add goclusterfs default - - # Set hostname - echo "goclusterfs-vm" | sudo tee "${MOUNT_POINT}/etc/hostname" > /dev/null - - # Configure fstab - cat < /dev/null -/dev/vda / ext4 defaults,noatime 0 1 -devtmpfs /dev devtmpfs defaults 0 0 -proc /proc proc defaults 0 0 -sysfs /sys sysfs defaults 0 0 -EOF - - # Unmount - sudo umount "${MOUNT_POINT}" - - # Convert to QCOW2 - log_info "Converting to QCOW2..." - qemu-img convert -f raw -O qcow2 "$raw_image" "$image_path" - rm -f "$raw_image" - - log_success "VM image created: ${image_path}" -} - -main() { - log_info "Building GoClusterFS VM image..." - - check_deps - mkdir -p "${RUNTIME_DIR}" "${OUTPUT_DIR}" - download_rootfs - build_image - - log_success "Build complete!" - log_info "Image location: ${OUTPUT_DIR}/${IMAGE_NAME}" -} - -main "$@" diff --git a/tests/vm/scripts/vm-entrypoint.sh b/tests/vm/scripts/vm-entrypoint.sh deleted file mode 100755 index 4327ad07..00000000 --- a/tests/vm/scripts/vm-entrypoint.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/sh -# GoClusterFS VM Entrypoint Script -# Runs inside the Alpine VM to start GoClusterFS -# -# This script: -# 1. Waits for the shared block device to be available -# 2. Optionally formats the filesystem (if FORMAT_DISK=true) -# 3. Mounts GoClusterFS at /mnt/goclusterfs -# 4. Handles graceful shutdown on signals - -set -e - -# Configuration -MOUNT_POINT="/mnt/goclusterfs" -DEVICE="/dev/vdb" # Second virtio disk (shared cluster storage) -GOCLUSTERFS="/usr/local/bin/goclusterfs-mount" -LOG_FILE="/var/log/goclusterfs.log" -READY_FILE="/tmp/goclusterfs-ready" - -# Read boot parameters -VM_ID="${VM_ID:-1}" -FORMAT_DISK="${FORMAT_DISK:-false}" - -# Logging -log() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${LOG_FILE}" -} - -log_error() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" | tee -a "${LOG_FILE}" >&2 -} - -# Wait for device -wait_for_device() { - local timeout=30 - local elapsed=0 - - log "Waiting for device ${DEVICE}..." - - while [ ! -b "${DEVICE}" ]; do - sleep 1 - elapsed=$((elapsed + 1)) - if [ $elapsed -ge $timeout ]; then - log_error "Device ${DEVICE} not found after ${timeout}s" - return 1 - fi - done - - log "Device ${DEVICE} is available" - return 0 -} - -# Signal handlers -cleanup() { - log "Received shutdown signal, unmounting..." - rm -f "${READY_FILE}" - fusermount -u "${MOUNT_POINT}" 2>/dev/null || true - log "Shutdown complete" - exit 0 -} - -trap cleanup SIGTERM SIGINT SIGQUIT - -# Main -main() { - log "GoClusterFS VM Entrypoint starting (VM_ID=${VM_ID}, FORMAT_DISK=${FORMAT_DISK})" - - # Ensure mount point exists - mkdir -p "${MOUNT_POINT}" - - # Wait for shared disk - if ! wait_for_device; then - log_error "Failed to find shared block device" - exit 1 - fi - - # Load FUSE module if needed - if [ ! -c /dev/fuse ]; then - log "Loading FUSE module..." - modprobe fuse || true - fi - - # Build mount command - MOUNT_ARGS="" - if [ "${FORMAT_DISK}" = "true" ]; then - MOUNT_ARGS="-format" - log "Will format filesystem on first mount" - fi - - log "Mounting GoClusterFS: ${GOCLUSTERFS} ${MOUNT_ARGS} ${MOUNT_POINT} ${DEVICE}" - - # Start GoClusterFS in background - ${GOCLUSTERFS} ${MOUNT_ARGS} "${MOUNT_POINT}" "${DEVICE}" >> "${LOG_FILE}" 2>&1 & - GOCLUSTERFS_PID=$! - - # Wait for mount to be ready - local timeout=30 - local elapsed=0 - while ! mountpoint -q "${MOUNT_POINT}"; do - if ! kill -0 $GOCLUSTERFS_PID 2>/dev/null; then - log_error "GoClusterFS process died unexpectedly" - cat "${LOG_FILE}" >&2 - exit 1 - fi - sleep 1 - elapsed=$((elapsed + 1)) - if [ $elapsed -ge $timeout ]; then - log_error "Mount point not ready after ${timeout}s" - exit 1 - fi - done - - log "GoClusterFS mounted successfully at ${MOUNT_POINT}" - - # Create ready marker for external health checks - touch "${READY_FILE}" - - # Wait for GoClusterFS process - wait $GOCLUSTERFS_PID -} - -main "$@" From 5cb7c3167037b7a85ba93b1fc716c2b7a0a59475 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 10:41:36 -0500 Subject: [PATCH 2/6] fix(vm): stop leaking VMs, isolate runs and keep VM logs - Traps on EXIT/INT/TERM stop the run's child processes and VMs. QEMU runs under setpriv --pdeathsig KILL, and the runner under --pdeathsig TERM, so a killed harness or runner cannot leave VMs or test processes behind. - Each run gets a private directory with its own SSH key and a copy of goclusterfs-mount taken at start. VMs accept only that key and listen on 127.0.0.1 ports from VM_SSH_BASE_PORT (default 2222). sshpass and the shared root password are gone. - VM setup no longer types long lines on the serial console, where the guest line editor wrapped and mangled them. The harness waits for the login prompt in the console log, logs in, and runs a setup script from a small read-only disk. A failed bring-up is retried once. - goclusterfs-mount writes its log to a virtio-serial port backed by a host file, so tests/vm/results/vmN.mount.log keeps everything up to a kill. Console and QEMU logs land there too, with boot, stop, kill and per-test markers. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- tests/vm/scripts/run-vm-tests.sh | 1 + tests/vm/vm-test-harness.sh | 501 ++++++++++++++++++++----------- 3 files changed, 321 insertions(+), 183 deletions(-) diff --git a/README.md b/README.md index 7e4c7d52..521b4b81 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ make test-multinode-vm make test-parity ``` -VM tests need an x86_64 Linux host with `/dev/kvm`, `qemu-system-x86_64`, and `sshpass`. QEMU 10+ uses io_uring, which counts against the locked-memory limit. With the common 8 MB default, QEMU fails with `Failed to initialize io_uring: Cannot allocate memory`. Raise the limit for the test user, then log in again: +VM tests need an x86_64 Linux host with `/dev/kvm`, `qemu-system-x86_64`, OpenBSD `nc` (Debian's `netcat-openbsd`) and `setpriv` from util-linux. VM N listens for SSH on `127.0.0.1:2222+N-1`; set `VM_SSH_BASE_PORT` to run two suites on one host. Each VM's console, QEMU and mount logs are written to `tests/vm/results/`, including logs from before a VM was killed. QEMU 10+ uses io_uring, which counts against the locked-memory limit. With the common 8 MB default, QEMU fails with `Failed to initialize io_uring: Cannot allocate memory`. Raise the limit for the test user, then log in again: ```bash echo "$USER - memlock unlimited" | sudo tee /etc/security/limits.d/90-qemu-memlock.conf diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 25df7901..b2247db8 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -54,6 +54,7 @@ run_test() { log_info "==========================================" log_info "Running test: $test_name" log_info "==========================================" + mark_vm_logs "Running test: $test_name" # Clean up test files from previous test to reclaim disk space cleanup_test_files 1 diff --git a/tests/vm/vm-test-harness.sh b/tests/vm/vm-test-harness.sh index 34d68c4e..c5e689ee 100755 --- a/tests/vm/vm-test-harness.sh +++ b/tests/vm/vm-test-harness.sh @@ -11,6 +11,10 @@ # ./vm-test-harness.sh --cleanup # Clean up all resources # ./vm-test-harness.sh --start-vm 1 # Start specific VM (for debugging) # ./vm-test-harness.sh --stop-vm 1 # Stop specific VM +# +# VM N listens for SSH on 127.0.0.1:$((VM_SSH_BASE_PORT + N - 1)), default base 2222. +# Each run gets its own SSH key and copy of goclusterfs-mount. VM console, QEMU and +# mount logs go to tests/vm/results/. set -euo pipefail @@ -34,22 +38,22 @@ SHARED_DISK_PATH="${VM_DIR}/cluster.img" SHARED_DISK_SIZE_MB=${VM_CLUSTER_SIZE_MB:-1024} # Network settings (each VM gets sequential ports) -SSH_BASE_PORT=2222 -QEMU_MONITOR_BASE_PORT=4444 +SSH_BASE_PORT=${VM_SSH_BASE_PORT:-2222} # VM resources VM_MEMORY="512M" VM_CPUS=1 +VM_BOOT_ATTEMPTS=2 + +# QEMU dies with the shell that started it, so a killed run cannot leave VMs behind +VM_REAPER=(setpriv --pdeathsig KILL --) # SSH settings -SSH_KEY_PATH="${VM_DIR}/test_key" SSH_USER="root" -SSH_TIMEOUT=30 -SSH_OPTIONS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5" +SSH_OPTIONS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5 -o BatchMode=yes -o IdentitiesOnly=yes" # Test settings DEFAULT_NODE_COUNT=2 -HEALTH_CHECK_RETRIES=60 HEALTH_CHECK_INTERVAL=2 # Colors for output @@ -93,21 +97,111 @@ get_cpu_type() { } # ============================================================================= -# SSH Key Management +# Run State # ============================================================================= -generate_ssh_key() { - if [[ -f "${SSH_KEY_PATH}" ]]; then +# A run owns a private directory holding its SSH key, a frozen copy of +# goclusterfs-mount and the VM setup disk. Child scripts inherit it. +init_vm_run() { + if [[ -n "${VM_RUN_DIR:-}" && -d "${VM_RUN_DIR}" ]]; then return 0 fi + if ! command -v setpriv > /dev/null; then + log_error "setpriv (util-linux) is required" + return 1 + fi + if [[ ! -x "${PROJECT_ROOT}/goclusterfs-mount" ]]; then + log_error "${PROJECT_ROOT}/goclusterfs-mount not found, run 'make build' first" + return 1 + fi - log_info "Generating SSH key for VM access..." - mkdir -p "${VM_DIR}" - ssh-keygen -t ed25519 -f "${SSH_KEY_PATH}" -N "" -q - log_success "SSH key generated: ${SSH_KEY_PATH}" + mkdir -p "${VM_DIR}" "${RESULTS_DIR}" + VM_RUN_DIR=$(mktemp -d "${VM_DIR}/run.XXXXXX") + VM_RUN_OWNER=$$ + export VM_RUN_DIR + + cp "${PROJECT_ROOT}/goclusterfs-mount" "${VM_RUN_DIR}/goclusterfs-mount" + ssh-keygen -t ed25519 -f "$(get_ssh_key)" -N "" -q -C "goclusterfs-vm-run" + write_setup_disk "${VM_RUN_DIR}/setup.img" + rm -f "${RESULTS_DIR}"/vm*.log + log_info "Run directory: ${VM_RUN_DIR}, VM logs: ${RESULTS_DIR}/vm*.log" +} + +start_vm_run() { + init_vm_run + trap on_vm_run_exit EXIT + trap 'exit 130' INT + trap 'exit 143' TERM +} + +on_vm_run_exit() { + local status=$? + trap - EXIT + trap '' INT TERM + set +e + stop_child_processes + kill_all_vms + if [[ "${VM_RUN_OWNER:-}" == "$$" ]]; then + rm -rf "${VM_RUN_DIR}" + fi + exit "$status" } -get_ssh_public_key() { - cat "${SSH_KEY_PATH}.pub" +kill_process_tree() { + local pid="$1" + local children child + children=$(pgrep -P "$pid" || true) + kill -TERM "$pid" 2>/dev/null || true + for child in $children; do + kill_process_tree "$child" + done +} + +stop_child_processes() { + local child + for child in $(pgrep -P $$ || true); do + kill_process_tree "$child" + done + wait 2>/dev/null || true +} + +get_ssh_key() { + echo "${VM_RUN_DIR}/id_ed25519" +} + +# Runs once per boot via "sh /dev/vdb". Typing long lines on the serial console +# is unreliable because the guest shell's line editor wraps them at 80 columns. +write_setup_disk() { + local image="$1" + { + echo "repo='${ALPINE_MIRROR}/v${ALPINE_VERSION}'" + echo "key='$(cat "$(get_ssh_key).pub")'" + echo "mountpoint='${GOCLUSTERFS_MOUNT}'" + cat <<'EOF' +set -x +setup-interfaces -a +ifup eth0 +printf '%s\n' "$repo/main" "$repo/community" > /etc/apk/repositories +tries=0 +until apk add --no-cache openssh fuse3 gcompat; do + tries=$((tries + 1)) + [ "$tries" -lt 5 ] || exit 1 + sleep 3 +done +modprobe fuse +modprobe virtio_console +for port in /sys/class/virtio-ports/*; do + [ "$(cat "$port/name")" = goclusterfs.log ] && ln -sf "/dev/${port##*/}" /dev/goclusterfs-log +done +[ -c /dev/goclusterfs-log ] || exit 1 +mkdir -p /root/.ssh "$mountpoint" +chmod 700 /root/.ssh +echo "$key" > /root/.ssh/authorized_keys +chmod 600 /root/.ssh/authorized_keys +ssh-keygen -A +rc-service sshd start +EOF + } > "$image" + truncate -s %512 "$image" } # ============================================================================= @@ -165,6 +259,21 @@ get_vm_monitor_socket() { echo "${VM_DIR}/vm${vm_id}.monitor.sock" } +get_vm_serial_socket() { + local vm_id="$1" + echo "${VM_DIR}/vm${vm_id}.sock" +} + +get_vm_console_log() { + local vm_id="$1" + echo "${RESULTS_DIR}/vm${vm_id}.console.log" +} + +get_vm_mount_log() { + local vm_id="$1" + echo "${RESULTS_DIR}/vm${vm_id}.mount.log" +} + get_vm_ip() { local vm_id="$1" # VMs use user-mode networking with port forwarding @@ -184,45 +293,69 @@ is_vm_running() { return 1 } -# Start a VM with Alpine live ISO +mark_vm_log() { + local vm_id="$1" + local text="$2" + [[ -d "${RESULTS_DIR}" ]] || return 0 + echo "=== $(date '+%F %T') $text" >> "$(get_vm_mount_log "$vm_id")" +} + +# Marks every VM mount log, e.g. with the name of the test about to run +mark_vm_logs() { + local log + for log in "${RESULTS_DIR}"/vm*.mount.log; do + if [[ -f "$log" ]]; then + echo "=== $(date '+%F %T') $*" >> "$log" + fi + done +} + +# Start a VM, retrying the whole bring-up once if it fails start_vm() { local vm_id="$1" - local format_disk="${2:-false}" # Whether this VM should format the shared disk + local attempt + + if is_vm_running "$vm_id"; then + log_info "VM $vm_id is already running" + return 0 + fi + + download_iso || return 1 + create_shared_disk || return 1 + + for attempt in $(seq 1 "$VM_BOOT_ATTEMPTS"); do + if boot_vm "$vm_id"; then + return 0 + fi + log_warn "VM $vm_id bring-up failed (attempt $attempt of $VM_BOOT_ATTEMPTS)" + kill_vm "$vm_id" + done + log_error "VM $vm_id failed to start" + return 1 +} +# Boot the Alpine live ISO, log in on the serial console and run the setup disk +boot_vm() { + local vm_id="$1" local ssh_port=$(get_vm_ssh_port "$vm_id") local monitor_socket=$(get_vm_monitor_socket "$vm_id") - local serial_socket="${VM_DIR}/vm${vm_id}.sock" + local serial_socket=$(get_vm_serial_socket "$vm_id") + local console_log=$(get_vm_console_log "$vm_id") local pid_file=$(get_vm_pid_file "$vm_id") - local log_file="${VM_DIR}/vm${vm_id}.log" local accelerator=$(get_accelerator) local cpu_type=$(get_cpu_type) - if is_vm_running "$vm_id"; then - log_info "VM $vm_id is already running" - return 0 - fi - # Clean up old sockets before starting (avoids "Address already in use") rm -f "$monitor_socket" "$serial_socket" - # Ensure prerequisites - download_iso - create_shared_disk - generate_ssh_key - - log_info "Starting VM $vm_id (SSH port: $ssh_port, format: $format_disk)..." + log_info "Starting VM $vm_id (SSH port: $ssh_port)..." log_info "Using accelerator: $accelerator, cpu: $cpu_type" - # Create cloud-init style seed for automatic configuration - local seed_dir="${VM_DIR}/seed${vm_id}" - mkdir -p "$seed_dir" + echo "=== $(date '+%F %T') VM $vm_id boot" >> "$console_log" + mark_vm_log "$vm_id" "VM $vm_id boot" + local console_offset=$(stat -c%s "$console_log") - # Create startup script that will be passed via kernel command line - local ssh_pubkey=$(get_ssh_public_key) - - # Start QEMU with Alpine live ISO - # The ISO boots to a login prompt - we configure it via serial console - qemu-system-x86_64 \ + "${VM_REAPER[@]}" qemu-system-x86_64 \ -name "goclusterfs-vm${vm_id}" \ -machine q35,accel=${accelerator} \ -cpu ${cpu_type} \ @@ -231,116 +364,109 @@ start_vm() { -cdrom "${ISO_PATH}" \ -boot d \ -drive "file=${SHARED_DISK_PATH},format=raw,if=virtio,cache=none,file.locking=off" \ - -netdev "user,id=net0,hostfwd=tcp::${ssh_port}-:22" \ + -drive "file=${VM_RUN_DIR}/setup.img,format=raw,if=virtio,readonly=on" \ + -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ -device "virtio-net-pci,netdev=net0" \ + -device virtio-serial-pci \ + -chardev "file,id=mountlog,path=$(get_vm_mount_log "$vm_id"),append=on" \ + -device "virtserialport,chardev=mountlog,name=goclusterfs.log" \ + -chardev "socket,id=serial0,path=${serial_socket},server=on,wait=off,logfile=${console_log},logappend=on" \ + -serial chardev:serial0 \ -monitor "unix:${monitor_socket},server,nowait" \ - -serial "unix:${serial_socket},server,nowait" \ -display none \ - -daemonize \ - -pidfile "${pid_file}" \ - 2>&1 || { - log_error "Failed to start VM $vm_id" - return 1 - } + >> "${RESULTS_DIR}/vm${vm_id}.qemu.log" 2>&1 & + local pid=$! + disown "$pid" + echo "$pid" > "$pid_file" + log_info "VM $vm_id started (PID: $pid)" + + if ! wait_for_console "$vm_id" "$console_offset" "login:" 120; then + report_boot_failure "$vm_id" "$console_offset" "no login prompt on the serial console" + return 1 + fi + if ! console_login "$vm_id" "$console_offset"; then + report_boot_failure "$vm_id" "$console_offset" "serial console login failed" + return 1 + fi - log_info "VM $vm_id started (PID: $(cat "$pid_file"))" + log_info "Running setup disk on VM $vm_id..." + console_send "$vm_id" "sh /dev/vdb" + if ! wait_for_ssh "$vm_id" 180; then + report_boot_failure "$vm_id" "$console_offset" "SSH not reachable after setup" + return 1 + fi - # Wait for VM to boot and configure it - sleep 10 - configure_vm "$vm_id" "$format_disk" + log_success "VM $vm_id configured and SSH verified" } -# Configure a running VM via serial console after boot -configure_vm() { +console_output_since() { local vm_id="$1" - local format_disk="${2:-false}" + local offset="$2" + tail -c +"$((offset + 1))" "$(get_vm_console_log "$vm_id")" 2>/dev/null || true +} + +console_send() { + local vm_id="$1" + local line="$2" + printf '%s\n' "$line" | nc -U -q 1 "$(get_vm_serial_socket "$vm_id")" > /dev/null 2>&1 || true +} - log_info "Configuring VM $vm_id via serial console..." +wait_for_console() { + local vm_id="$1" + local offset="$2" + local text="$3" + local timeout="$4" + local elapsed=0 - # Always configure via serial - QEMU port forwarding makes port appear open - # even before sshd is running, so we can't use nc -z to detect readiness - configure_vm_via_serial "$vm_id" "$format_disk" + until [[ "$(console_output_since "$vm_id" "$offset")" == *"$text"* ]]; do + if (( elapsed >= timeout )) || ! is_vm_running "$vm_id"; then + return 1 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done } -# Configure VM by sending commands through serial socket -configure_vm_via_serial() { +console_login() { local vm_id="$1" - local format_disk="$2" - local serial_sock="${VM_DIR}/vm${vm_id}.sock" - local ssh_pubkey=$(get_ssh_public_key) - - # Helper to send a command via serial socket - send_serial_cmd() { - local cmd="$1" - local wait="${2:-1}" - # Send command with newline - echo "$cmd" | nc -U -w 1 "$serial_sock" 2>/dev/null || true - sleep "$wait" - } - - # Wait for login prompt - log_info "Waiting for VM $vm_id login prompt..." - sleep 15 # Wait for boot - - # Login as root (no password on Alpine live) - log_info "Logging into VM $vm_id..." - send_serial_cmd "root" 2 - - # Configure networking - log_info "Configuring networking..." - send_serial_cmd "setup-interfaces -a" 3 - send_serial_cmd "ifup eth0" 2 - - # Configure Alpine repositories (live ISO only has CD-ROM packages) - log_info "Configuring Alpine repositories..." - send_serial_cmd "echo 'https://dl-cdn.alpinelinux.org/alpine/v3.19/main' > /etc/apk/repositories" 1 - send_serial_cmd "echo 'https://dl-cdn.alpinelinux.org/alpine/v3.19/community' >> /etc/apk/repositories" 1 - send_serial_cmd "apk update" 10 - - # Install and configure SSH - log_info "Installing SSH..." - send_serial_cmd "apk add --no-cache openssh" 15 - - log_info "Configuring SSH..." - send_serial_cmd "sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config" 1 - send_serial_cmd "sed -i 's/#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config" 1 - send_serial_cmd "ssh-keygen -A" 3 - send_serial_cmd "rc-service sshd start" 3 - - # Set root password - log_info "Setting root password..." - send_serial_cmd "echo 'root:testpass123' | chpasswd" 1 - - # Set up SSH key - log_info "Installing SSH key..." - send_serial_cmd "mkdir -p /root/.ssh" 1 - send_serial_cmd "chmod 700 /root/.ssh" 1 - send_serial_cmd "echo '$ssh_pubkey' > /root/.ssh/authorized_keys" 1 - send_serial_cmd "chmod 600 /root/.ssh/authorized_keys" 1 - - # Verify SSH is actually working before returning - # This prevents race conditions where start_vm returns before SSH is ready - log_info "Verifying SSH connectivity for VM $vm_id..." - local ssh_port=$(get_vm_ssh_port "$vm_id") - local ssh_timeout=60 - local ssh_elapsed=0 + local offset="$2" + local attempt - while [[ $ssh_elapsed -lt $ssh_timeout ]]; do - if sshpass -p "testpass123" ssh ${SSH_OPTIONS} -p "$ssh_port" "${SSH_USER}@127.0.0.1" "true" 2>/dev/null; then - log_success "VM $vm_id configured and SSH verified" + for attempt in 1 2 3; do + console_send "$vm_id" "root" + if wait_for_console "$vm_id" "$offset" ":~# " 15; then return 0 fi - sleep 2 - ssh_elapsed=$((ssh_elapsed + 2)) done - - log_error "VM $vm_id: SSH verification failed after ${ssh_timeout}s" return 1 } -# Alias for backward compatibility -configure_vm_via_monitor() { - configure_vm_via_serial "$@" +wait_for_ssh() { + local vm_id="$1" + local timeout="$2" + local elapsed=0 + + until ssh_run "$vm_id" true < /dev/null 2>/dev/null; do + if (( elapsed >= timeout )) || ! is_vm_running "$vm_id"; then + return 1 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done +} + +report_boot_failure() { + local vm_id="$1" + local offset="$2" + local reason="$3" + + log_error "VM $vm_id: $reason" + if ! is_vm_running "$vm_id"; then + log_error "QEMU exited, see ${RESULTS_DIR}/vm${vm_id}.qemu.log:" + tail -n 5 "${RESULTS_DIR}/vm${vm_id}.qemu.log" >&2 || true + fi + log_error "Console output (full log: $(get_vm_console_log "$vm_id")):" + console_output_since "$vm_id" "$offset" | tail -n 20 >&2 } # ============================================================================= @@ -355,33 +481,22 @@ setup_goclusterfs() { log_info "Setting up GoClusterFS on VM $vm_id (format=$format_disk)..." - # Copy the binary to the VM log_info "Copying goclusterfs-mount binary..." - sshpass -p "testpass123" scp ${SSH_OPTIONS} -P "$ssh_port" \ - "${PROJECT_ROOT}/goclusterfs-mount" \ - "root@127.0.0.1:/usr/local/bin/goclusterfs-mount" - - # Install fuse3 and gcompat (glibc compatibility for binary compiled on glibc systems) - log_info "Installing fuse3 and gcompat..." - ssh_run_password "$vm_id" "apk add --no-cache fuse3 gcompat" - - # Load FUSE kernel module (required before mounting any FUSE filesystem) - log_info "Loading FUSE kernel module..." - ssh_run_password "$vm_id" "modprobe fuse" - - # Create mount point - ssh_run_password "$vm_id" "mkdir -p /mnt/goclusterfs" + scp ${SSH_OPTIONS} -i "$(get_ssh_key)" -P "$ssh_port" \ + "${VM_RUN_DIR}/goclusterfs-mount" \ + "${SSH_USER}@127.0.0.1:/usr/local/bin/goclusterfs-mount" # Mount GoClusterFS - local mount_args="-mount /mnt/goclusterfs -device /dev/vda" + local mount_args="-mount ${GOCLUSTERFS_MOUNT} -device /dev/vda" if [[ "$format_disk" == "true" ]]; then log_info "Formatting shared disk on VM $vm_id..." mount_args="$mount_args -format" fi + # The log device streams to the host, so the log survives VM kills and restarts log_info "Mounting GoClusterFS..." - ssh_run_password "$vm_id" \ - "nohup /usr/local/bin/goclusterfs-mount $mount_args > /var/log/goclusterfs.log 2>&1 &" + ssh_run "$vm_id" \ + "nohup /usr/local/bin/goclusterfs-mount $mount_args < /dev/null > /dev/goclusterfs-log 2>&1 &" # Wait for mount to be ready wait_for_goclusterfs_mount "$vm_id" @@ -396,7 +511,7 @@ wait_for_goclusterfs_mount() { log_info "Waiting for GoClusterFS mount on VM $vm_id..." while [[ $elapsed -lt $timeout ]]; do - if ssh_run_password "$vm_id" "mountpoint -q /mnt/goclusterfs" 2>/dev/null; then + if ssh_run "$vm_id" "mountpoint -q ${GOCLUSTERFS_MOUNT}" 2>/dev/null; then log_success "GoClusterFS mounted on VM $vm_id" return 0 fi @@ -404,8 +519,8 @@ wait_for_goclusterfs_mount() { elapsed=$((elapsed + 1)) done - log_error "GoClusterFS mount timeout on VM $vm_id" - ssh_run_password "$vm_id" "cat /var/log/goclusterfs.log" 2>/dev/null || true + log_error "GoClusterFS mount timeout on VM $vm_id, last lines of $(get_vm_mount_log "$vm_id"):" + tail -n 20 "$(get_vm_mount_log "$vm_id")" >&2 || true return 1 } @@ -420,10 +535,11 @@ stop_vm() { fi log_info "Stopping VM $vm_id gracefully..." + mark_vm_log "$vm_id" "VM $vm_id stop" # Try graceful shutdown via monitor (Unix socket) local monitor_socket=$(get_vm_monitor_socket "$vm_id") - echo "system_powerdown" | nc -U -q1 "$monitor_socket" 2>/dev/null || true + echo "system_powerdown" | nc -U -q1 "$monitor_socket" > /dev/null 2>&1 || true # Wait for graceful shutdown local timeout=30 @@ -440,7 +556,7 @@ stop_vm() { rm -f "$pid_file" # Clean up sockets - rm -f "$(get_vm_monitor_socket "$vm_id")" "${VM_DIR}/vm${vm_id}.sock" + rm -f "$(get_vm_monitor_socket "$vm_id")" "$(get_vm_serial_socket "$vm_id")" log_success "VM $vm_id stopped" } @@ -453,13 +569,30 @@ kill_vm() { local pid=$(cat "$pid_file") kill -9 "$pid" 2>/dev/null || true rm -f "$pid_file" + mark_vm_log "$vm_id" "VM $vm_id killed" fi # Clean up sockets - rm -f "$(get_vm_monitor_socket "$vm_id")" "${VM_DIR}/vm${vm_id}.sock" + rm -f "$(get_vm_monitor_socket "$vm_id")" "$(get_vm_serial_socket "$vm_id")" log_info "VM $vm_id killed" } +vm_ids_with_pid_files() { + local pid_file + for pid_file in "${VM_DIR}"/vm*.pid; do + if [[ -f "$pid_file" ]]; then + basename "$pid_file" | sed 's/vm\([0-9]*\).pid/\1/' + fi + done +} + +kill_all_vms() { + local vm_id + for vm_id in $(vm_ids_with_pid_files); do + kill_vm "$vm_id" + done +} + # ============================================================================= # SSH Functions for Tests # ============================================================================= @@ -468,28 +601,24 @@ ssh_run() { shift local ssh_port=$(get_vm_ssh_port "$vm_id") - ssh ${SSH_OPTIONS} -i "${SSH_KEY_PATH}" -p "$ssh_port" "${SSH_USER}@127.0.0.1" "$@" + ssh ${SSH_OPTIONS} -i "$(get_ssh_key)" -p "$ssh_port" "${SSH_USER}@127.0.0.1" "$@" } +# VMs accept only the run's SSH key; kept for existing callers ssh_run_password() { - local vm_id="$1" - shift - local ssh_port=$(get_vm_ssh_port "$vm_id") - - sshpass -p "testpass123" ssh ${SSH_OPTIONS} -p "$ssh_port" "${SSH_USER}@127.0.0.1" "$@" + ssh_run "$@" } wait_vm_healthy() { local vm_id="$1" local timeout="${2:-120}" - local ssh_port=$(get_vm_ssh_port "$vm_id") log_info "Waiting for VM $vm_id to be healthy..." local elapsed=0 while [[ $elapsed -lt $timeout ]]; do # First check if SSH is reachable at all - if sshpass -p "testpass123" ssh ${SSH_OPTIONS} -p "$ssh_port" "${SSH_USER}@127.0.0.1" "true" 2>/dev/null; then + if ssh_run "$vm_id" "true" 2>/dev/null; then log_success "VM $vm_id is healthy (SSH reachable)" return 0 fi @@ -508,12 +637,11 @@ wait_vm_healthy() { # restarts the VM and waits for it to become healthy. ensure_vm_running() { local vm_id="$1" - local ssh_port=$(get_vm_ssh_port "$vm_id") # Check if VM is reachable via SSH - if sshpass -p "testpass123" ssh ${SSH_OPTIONS} -p "$ssh_port" "${SSH_USER}@127.0.0.1" "true" 2>/dev/null; then + if ssh_run "$vm_id" "true" 2>/dev/null; then # VM is healthy, check if GoClusterFS is mounted - if sshpass -p "testpass123" ssh ${SSH_OPTIONS} -p "$ssh_port" "${SSH_USER}@127.0.0.1" "mountpoint -q /mnt/goclusterfs" 2>/dev/null; then + if ssh_run "$vm_id" "mountpoint -q ${GOCLUSTERFS_MOUNT}" 2>/dev/null; then return 0 # Fully healthy fi # SSH works but GoClusterFS not mounted - remount @@ -557,21 +685,21 @@ write_file_on_vm() { local filename="$2" local content="$3" - ssh_run_password "$vm_id" "echo '$content' > ${GOCLUSTERFS_MOUNT}/$filename" + ssh_run "$vm_id" "echo '$content' > ${GOCLUSTERFS_MOUNT}/$filename" } read_file_on_vm() { local vm_id="$1" local filename="$2" - ssh_run_password "$vm_id" "cat ${GOCLUSTERFS_MOUNT}/$filename" + ssh_run "$vm_id" "cat ${GOCLUSTERFS_MOUNT}/$filename" } file_exists_on_vm() { local vm_id="$1" local filename="$2" - ssh_run_password "$vm_id" "test -f ${GOCLUSTERFS_MOUNT}/$filename" + ssh_run "$vm_id" "test -f ${GOCLUSTERFS_MOUNT}/$filename" } # Clean up test files from the GoClusterFS filesystem @@ -583,7 +711,7 @@ cleanup_test_files() { log_info "Cleaning up test files from GoClusterFS..." # Check if VM is reachable and GoClusterFS is mounted - if ! ssh_run_password "$vm_id" "mountpoint -q ${GOCLUSTERFS_MOUNT}" 2>/dev/null; then + if ! ssh_run "$vm_id" "mountpoint -q ${GOCLUSTERFS_MOUNT}" 2>/dev/null; then log_warn "GoClusterFS not mounted on VM $vm_id, skipping cleanup" return 0 fi @@ -591,14 +719,14 @@ cleanup_test_files() { # Remove all files and directories from mount point # Use find to avoid "argument list too long" errors with many files # Also handles hidden files (.*) safely - ssh_run_password "$vm_id" "find ${GOCLUSTERFS_MOUNT} -mindepth 1 -delete 2>/dev/null || rm -rf ${GOCLUSTERFS_MOUNT}/* ${GOCLUSTERFS_MOUNT}/.[!.]* 2>/dev/null" || true + ssh_run "$vm_id" "find ${GOCLUSTERFS_MOUNT} -mindepth 1 -delete 2>/dev/null || rm -rf ${GOCLUSTERFS_MOUNT}/* ${GOCLUSTERFS_MOUNT}/.[!.]* 2>/dev/null" || true # Sync to ensure blocks are freed - ssh_run_password "$vm_id" "sync" + ssh_run "$vm_id" "sync" # Log free space for debugging local free_space - free_space=$(ssh_run_password "$vm_id" "df -h ${GOCLUSTERFS_MOUNT} 2>/dev/null | tail -1 | awk '{print \$4}'" 2>/dev/null) || true + free_space=$(ssh_run "$vm_id" "df -h ${GOCLUSTERFS_MOUNT} 2>/dev/null | tail -1 | awk '{print \$4}'" 2>/dev/null) || true if [[ -n "$free_space" ]]; then log_info "Free space after cleanup: $free_space" fi @@ -612,12 +740,9 @@ cleanup_test_files() { cleanup() { log_info "Cleaning up all VMs..." - # Stop all VMs - for pid_file in "${VM_DIR}"/vm*.pid; do - if [[ -f "$pid_file" ]]; then - local vm_id=$(basename "$pid_file" | sed 's/vm\([0-9]*\).pid/\1/') - stop_vm "$vm_id" 2>/dev/null || true - fi + local vm_id + for vm_id in $(vm_ids_with_pid_files); do + stop_vm "$vm_id" 2>/dev/null || true done log_success "Cleanup complete" @@ -640,8 +765,9 @@ run_tests() { log_info "Running VM test suite with $node_count nodes..." - # Run the test script - "${SCRIPT_DIR}/scripts/run-vm-tests.sh" "$node_count" + # In the background so traps run while waiting; the runner gets SIGTERM if this shell dies + setpriv --pdeathsig TERM -- "${SCRIPT_DIR}/scripts/run-vm-tests.sh" "$node_count" & + wait $! } # ============================================================================= @@ -655,15 +781,20 @@ main() { shift local nodes="${1:-$DEFAULT_NODE_COUNT}" [[ "$nodes" == --nodes=* ]] && nodes="${nodes#--nodes=}" + start_vm_run run_tests "$nodes" ;; --nodes=*) local nodes="${command#--nodes=}" + start_vm_run run_tests "$nodes" ;; --start-vm) shift - start_vm "$1" "${2:-false}" + VM_REAPER=(setsid) + init_vm_run + start_vm "$1" + log_info "SSH: ssh -i $(get_ssh_key) -p $(get_vm_ssh_port "$1") root@127.0.0.1" ;; --stop-vm) shift @@ -675,6 +806,7 @@ main() { ;; --cleanup) cleanup + rm -rf "${VM_DIR}"/run.* ;; --cleanup-all) cleanup_all @@ -684,12 +816,15 @@ main() { echo "" echo "Usage:" echo " $0 --run-tests [--nodes=N] Run tests with N VMs (default: 2)" - echo " $0 --start-vm ID [format] Start specific VM" + echo " $0 --start-vm ID Start specific VM" echo " $0 --stop-vm ID Stop specific VM" echo " $0 --kill-vm ID Force kill VM" echo " $0 --cleanup Stop all VMs" echo " $0 --cleanup-all Remove all runtime files" echo " $0 --help Show this help" + echo "" + echo "Environment:" + echo " VM_SSH_BASE_PORT Host SSH port of VM 1 (default: 2222)" ;; *) log_error "Unknown command: $command" @@ -699,7 +834,9 @@ main() { esac } -# Run if executed directly +# Run if executed directly; a sourcing test runner joins the run or starts its own if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then main "$@" +else + start_vm_run fi From de3012a9b1887a0a42abb155ada6fd420bf9f927 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 14:28:01 -0500 Subject: [PATCH 3/6] fix(vm): ignore stale VM pid files and clean up after a killed harness A pid file left by a killed run could name a reused PID, and kill_vm would then SIGKILL an unrelated process. is_vm_running now checks that the PID is still that VM's QEMU, kill_vm only kills a running VM, and stop_vm removes stale pid files and sockets. The runner removes the run directory when the harness that created it is gone. Co-Authored-By: Claude Opus 5.5 --- tests/vm/vm-test-harness.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/vm/vm-test-harness.sh b/tests/vm/vm-test-harness.sh index c5e689ee..44854ba5 100755 --- a/tests/vm/vm-test-harness.sh +++ b/tests/vm/vm-test-harness.sh @@ -117,7 +117,7 @@ init_vm_run() { mkdir -p "${VM_DIR}" "${RESULTS_DIR}" VM_RUN_DIR=$(mktemp -d "${VM_DIR}/run.XXXXXX") VM_RUN_OWNER=$$ - export VM_RUN_DIR + export VM_RUN_DIR VM_RUN_OWNER cp "${PROJECT_ROOT}/goclusterfs-mount" "${VM_RUN_DIR}/goclusterfs-mount" ssh-keygen -t ed25519 -f "$(get_ssh_key)" -N "" -q -C "goclusterfs-vm-run" @@ -140,7 +140,7 @@ on_vm_run_exit() { set +e stop_child_processes kill_all_vms - if [[ "${VM_RUN_OWNER:-}" == "$$" ]]; then + if [[ "${VM_RUN_OWNER:-}" == "$$" ]] || ! kill -0 "${VM_RUN_OWNER:-$$}" 2>/dev/null; then rm -rf "${VM_RUN_DIR}" fi exit "$status" @@ -284,9 +284,11 @@ is_vm_running() { local vm_id="$1" local pid_file=$(get_vm_pid_file "$vm_id") + # A stale pid file may name a reused PID, so check that it is still this VM's QEMU if [[ -f "$pid_file" ]]; then local pid=$(cat "$pid_file") - if kill -0 "$pid" 2>/dev/null; then + local cmdline=$(tr '\0' ' ' < "/proc/${pid}/cmdline" 2>/dev/null) + if [[ "$cmdline" == *"-name goclusterfs-vm${vm_id} "* ]]; then return 0 fi fi @@ -531,6 +533,7 @@ stop_vm() { if ! is_vm_running "$vm_id"; then log_info "VM $vm_id is not running" + rm -f "$pid_file" "$(get_vm_monitor_socket "$vm_id")" "$(get_vm_serial_socket "$vm_id")" return 0 fi @@ -565,12 +568,11 @@ kill_vm() { local vm_id="$1" local pid_file=$(get_vm_pid_file "$vm_id") - if [[ -f "$pid_file" ]]; then - local pid=$(cat "$pid_file") - kill -9 "$pid" 2>/dev/null || true - rm -f "$pid_file" + if is_vm_running "$vm_id"; then + kill -9 "$(cat "$pid_file")" 2>/dev/null || true mark_vm_log "$vm_id" "VM $vm_id killed" fi + rm -f "$pid_file" # Clean up sockets rm -f "$(get_vm_monitor_socket "$vm_id")" "$(get_vm_serial_socket "$vm_id")" From 7a243f8574a0d7eaf5866227d56db8c93ab6519d Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 15:35:36 -0500 Subject: [PATCH 4/6] fix(vm): keep /var/log/goclusterfs.log in the VM The journal tests from #15 grep the in-VM mount log. Write it with tee and stream the same output to the host log device. Co-Authored-By: Claude Opus 5.5 --- tests/vm/vm-test-harness.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/vm/vm-test-harness.sh b/tests/vm/vm-test-harness.sh index 44854ba5..8af48ef2 100755 --- a/tests/vm/vm-test-harness.sh +++ b/tests/vm/vm-test-harness.sh @@ -495,10 +495,10 @@ setup_goclusterfs() { mount_args="$mount_args -format" fi - # The log device streams to the host, so the log survives VM kills and restarts + # /var/log/goclusterfs.log is lost when the VM dies; the log device keeps a copy on the host log_info "Mounting GoClusterFS..." ssh_run "$vm_id" \ - "nohup /usr/local/bin/goclusterfs-mount $mount_args < /dev/null > /dev/goclusterfs-log 2>&1 &" + "nohup /usr/local/bin/goclusterfs-mount $mount_args < /dev/null 2>&1 | tee /var/log/goclusterfs.log > /dev/goclusterfs-log 2>/dev/null &" # Wait for mount to be ready wait_for_goclusterfs_mount "$vm_id" From d04e198702f83c964a658ec9a16ce08c33553ec4 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 15:45:26 -0500 Subject: [PATCH 5/6] fix(vm): one run per checkout and pin the Alpine ISO hash - init_vm_run takes a lock on runtime/harness.lock and refuses a second run in the same checkout, which would share pid files and ports. QEMU does not inherit the lock, so debug VMs from --start-vm do not hold it. A new run stops VMs left in the checkout by an earlier run. - --cleanup keeps the run directory of a run that still holds the lock. - download_iso checks the ISO against a pinned SHA-256 and removes a file that does not match. Co-Authored-By: Claude Opus 5.5 --- tests/vm/vm-test-harness.sh | 50 +++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/tests/vm/vm-test-harness.sh b/tests/vm/vm-test-harness.sh index 8af48ef2..53e0408b 100755 --- a/tests/vm/vm-test-harness.sh +++ b/tests/vm/vm-test-harness.sh @@ -13,7 +13,8 @@ # ./vm-test-harness.sh --stop-vm 1 # Stop specific VM # # VM N listens for SSH on 127.0.0.1:$((VM_SSH_BASE_PORT + N - 1)), default base 2222. -# Each run gets its own SSH key and copy of goclusterfs-mount. VM console, QEMU and +# A lock allows one run per checkout; runs from different checkouts on one host +# need different VM_SSH_BASE_PORT values. Each run gets its own SSH key and copy of goclusterfs-mount. VM console, QEMU and # mount logs go to tests/vm/results/. set -euo pipefail @@ -32,6 +33,7 @@ ALPINE_MIRROR="https://dl-cdn.alpinelinux.org/alpine" ISO_NAME="alpine-virt-${ALPINE_VERSION}.0-x86_64.iso" ISO_URL="${ALPINE_MIRROR}/v${ALPINE_VERSION}/releases/x86_64/${ISO_NAME}" ISO_PATH="${VM_DIR}/${ISO_NAME}" +ISO_SHA256="95ed37b1e54b04aad1c6ff54a5fd60d904363a2a8810c3e3dfe4c2a29e94056b" # Shared storage (configurable via VM_CLUSTER_SIZE_MB environment variable) SHARED_DISK_PATH="${VM_DIR}/cluster.img" @@ -115,9 +117,14 @@ init_vm_run() { fi mkdir -p "${VM_DIR}" "${RESULTS_DIR}" + exec {VM_RUN_LOCK_FD}> "${VM_DIR}/harness.lock" + if ! flock -n "$VM_RUN_LOCK_FD"; then + log_error "Another VM run is using ${VM_DIR}. Wait for it, stop it with --cleanup, or run from another checkout with a different VM_SSH_BASE_PORT" + return 1 + fi VM_RUN_DIR=$(mktemp -d "${VM_DIR}/run.XXXXXX") VM_RUN_OWNER=$$ - export VM_RUN_DIR VM_RUN_OWNER + export VM_RUN_DIR VM_RUN_OWNER VM_RUN_LOCK_FD cp "${PROJECT_ROOT}/goclusterfs-mount" "${VM_RUN_DIR}/goclusterfs-mount" ssh-keygen -t ed25519 -f "$(get_ssh_key)" -N "" -q -C "goclusterfs-vm-run" @@ -131,6 +138,23 @@ start_vm_run() { trap on_vm_run_exit EXIT trap 'exit 130' INT trap 'exit 143' TERM + if [[ "${VM_RUN_OWNER}" == "$$" && -n "$(vm_ids_with_pid_files)" ]]; then + log_warn "Stopping VMs left in ${VM_DIR} by an earlier run" + kill_all_vms + fi +} + +# Removes run directories unless a run in this checkout still holds the lock +remove_idle_run_dirs() { + local lock_fd + [[ -d "${VM_DIR}" ]] || return 0 + exec {lock_fd}> "${VM_DIR}/harness.lock" + if flock -n "$lock_fd"; then + rm -rf "${VM_DIR}"/run.* + else + log_warn "A VM run is active in ${VM_DIR}, keeping its run directory" + fi + exec {lock_fd}>&- } on_vm_run_exit() { @@ -210,13 +234,18 @@ EOF download_iso() { if [[ -f "${ISO_PATH}" ]]; then log_info "Alpine ISO already downloaded" - return 0 + else + log_info "Downloading Alpine Linux ISO..." + mkdir -p "${VM_DIR}" + wget -q --show-progress -O "${ISO_PATH}" "${ISO_URL}" + log_success "Downloaded: ${ISO_PATH}" fi - log_info "Downloading Alpine Linux ISO..." - mkdir -p "${VM_DIR}" - wget -q --show-progress -O "${ISO_PATH}" "${ISO_URL}" - log_success "Downloaded: ${ISO_PATH}" + if ! echo "${ISO_SHA256} ${ISO_PATH}" | sha256sum -c --status; then + log_error "${ISO_PATH} does not match the pinned SHA-256; removed it, rerun to download again" + rm -f "${ISO_PATH}" + return 1 + fi } # ============================================================================= @@ -376,7 +405,7 @@ boot_vm() { -serial chardev:serial0 \ -monitor "unix:${monitor_socket},server,nowait" \ -display none \ - >> "${RESULTS_DIR}/vm${vm_id}.qemu.log" 2>&1 & + {VM_RUN_LOCK_FD}>&- >> "${RESULTS_DIR}/vm${vm_id}.qemu.log" 2>&1 & local pid=$! disown "$pid" echo "$pid" > "$pid_file" @@ -808,7 +837,7 @@ main() { ;; --cleanup) cleanup - rm -rf "${VM_DIR}"/run.* + remove_idle_run_dirs ;; --cleanup-all) cleanup_all @@ -826,7 +855,8 @@ main() { echo " $0 --help Show this help" echo "" echo "Environment:" - echo " VM_SSH_BASE_PORT Host SSH port of VM 1 (default: 2222)" + echo " VM_SSH_BASE_PORT Host SSH port of VM 1 (default: 2222); use a different" + echo " value for each checkout that runs VMs at the same time" ;; *) log_error "Unknown command: $command" From c65734c5b517a1bdc6f60f64742653d322a05c7a Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 24 Oct 2026 15:52:52 -0500 Subject: [PATCH 6/6] fix(vm): install attr for the snapshot tests from #16 Co-Authored-By: Claude Opus 5.5 --- tests/vm/vm-test-harness.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/vm/vm-test-harness.sh b/tests/vm/vm-test-harness.sh index 53e0408b..7bd60576 100755 --- a/tests/vm/vm-test-harness.sh +++ b/tests/vm/vm-test-harness.sh @@ -206,7 +206,7 @@ setup-interfaces -a ifup eth0 printf '%s\n' "$repo/main" "$repo/community" > /etc/apk/repositories tries=0 -until apk add --no-cache openssh fuse3 gcompat; do +until apk add --no-cache openssh fuse3 gcompat attr; do tries=$((tries + 1)) [ "$tries" -lt 5 ] || exit 1 sleep 3