diff --git a/README.md b/README.md index 7e4c7d52..521b4b81 100644 --- a/README.md +++ b/README.md @@ -208,7 +208,7 @@ make test-multinode-vm make test-parity ``` -VM tests need an x86_64 Linux host with `/dev/kvm`, `qemu-system-x86_64`, and `sshpass`. QEMU 10+ uses io_uring, which counts against the locked-memory limit. With the common 8 MB default, QEMU fails with `Failed to initialize io_uring: Cannot allocate memory`. Raise the limit for the test user, then log in again: +VM tests need an x86_64 Linux host with `/dev/kvm`, `qemu-system-x86_64`, OpenBSD `nc` (Debian's `netcat-openbsd`) and `setpriv` from util-linux. VM N listens for SSH on `127.0.0.1:2222+N-1`; set `VM_SSH_BASE_PORT` to run two suites on one host. Each VM's console, QEMU and mount logs are written to `tests/vm/results/`, including logs from before a VM was killed. QEMU 10+ uses io_uring, which counts against the locked-memory limit. With the common 8 MB default, QEMU fails with `Failed to initialize io_uring: Cannot allocate memory`. Raise the limit for the test user, then log in again: ```bash echo "$USER - memlock unlimited" | sudo tee /etc/security/limits.d/90-qemu-memlock.conf diff --git a/tests/vm/packer/alpine.pkr.hcl b/tests/vm/packer/alpine.pkr.hcl deleted file mode 100644 index 433f446f..00000000 --- a/tests/vm/packer/alpine.pkr.hcl +++ /dev/null @@ -1,184 +0,0 @@ -# GoClusterFS Alpine VM Image -# Packer template using pre-built Alpine cloud image (more reliable than ISO install) -# -# Usage: -# export GOCLUSTERFS_BINARY=/path/to/goclusterfs-mount -# export SSH_PUBLIC_KEY="ssh-ed25519 AAAA..." -# export OUTPUT_DIR=/path/to/output -# packer build alpine.pkr.hcl - -packer { - required_plugins { - qemu = { - version = ">= 1.0.0" - source = "github.com/hashicorp/qemu" - } - } -} - -# Variables -variable "alpine_version" { - type = string - default = "3.19" -} - -variable "output_directory" { - type = string - default = env("OUTPUT_DIR") -} - -variable "goclusterfs_binary" { - type = string - default = env("GOCLUSTERFS_BINARY") -} - -variable "ssh_public_key" { - type = string - default = env("SSH_PUBLIC_KEY") -} - -variable "disk_size" { - type = string - default = "1G" -} - -variable "memory" { - type = string - default = "512" -} - -# Local variables -locals { - # Alpine provides pre-built cloud images in QCOW2 format - cloud_image_url = "https://dl-cdn.alpinelinux.org/alpine/v${var.alpine_version}/releases/x86_64/alpine-virt-${var.alpine_version}.0-x86_64.iso" - output_dir = var.output_directory != "" ? var.output_directory : "${path.root}/../runtime/packer-output" -} - -# QEMU builder - use a simpler approach with direct console access -source "qemu" "alpine" { - iso_url = local.cloud_image_url - iso_checksum = "file:https://dl-cdn.alpinelinux.org/alpine/v${var.alpine_version}/releases/x86_64/alpine-virt-${var.alpine_version}.0-x86_64.iso.sha256" - output_directory = local.output_dir - vm_name = "goclusterfs-alpine.qcow2" - - # VM configuration - disk_size = var.disk_size - memory = var.memory - cpus = 1 - accelerator = "kvm" - format = "qcow2" - disk_interface = "virtio" - net_device = "virtio-net" - - # Boot settings - simplified for reliability - boot_wait = "30s" - boot_command = [ - # Login as root (no password on live system) - "root", - - # Set up basic networking - "setup-interfaces -a", - "ifup eth0", - - # Set root password for SSH - "echo 'root:testpass123' | chpasswd", - - # Enable and configure SSH - "apk add openssh", - "sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config", - "sed -i 's/#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config", - "rc-service sshd start", - "rc-update add sshd", - - # Set up disk for installation - "setup-disk -m sys -s 0 /dev/vda", - "y", - - # Reboot into installed system - "reboot", - - # Login after reboot - "root", - "testpass123", - - # Re-enable SSH after reboot - "rc-service sshd start", - ] - - # SSH configuration for provisioning - ssh_username = "root" - ssh_password = "testpass123" - ssh_timeout = "15m" - shutdown_command = "poweroff" - - # Headless mode - headless = true - - # QEMU display - display = "none" -} - -# Build -build { - sources = ["source.qemu.alpine"] - - # Install required packages - provisioner "shell" { - inline = [ - "apk update", - "apk add fuse3 fuse3-dev util-linux procps", - "modprobe fuse || true", - "echo 'fuse' >> /etc/modules", - ] - } - - # Set up SSH key authentication - provisioner "shell" { - inline = [ - "mkdir -p /root/.ssh", - "chmod 700 /root/.ssh", - "echo '${var.ssh_public_key}' > /root/.ssh/authorized_keys", - "chmod 600 /root/.ssh/authorized_keys", - ] - } - - # Copy GoClusterFS binary - provisioner "file" { - source = var.goclusterfs_binary - destination = "/usr/local/bin/goclusterfs-mount" - } - - provisioner "shell" { - inline = [ - "chmod +x /usr/local/bin/goclusterfs-mount", - ] - } - - # Copy entrypoint script - provisioner "file" { - source = "${path.root}/../scripts/vm-entrypoint.sh" - destination = "/usr/local/bin/vm-entrypoint.sh" - } - - provisioner "shell" { - inline = [ - "chmod +x /usr/local/bin/vm-entrypoint.sh", - ] - } - - # Set up auto-start of GoClusterFS - provisioner "shell" { - inline = [ - # Create init script for GoClusterFS - "cat > /etc/init.d/goclusterfs << 'INITEOF'\n#!/sbin/openrc-run\n\ndepend() {\n need localmount\n after modules\n}\n\nstart() {\n ebegin \"Starting GoClusterFS\"\n /usr/local/bin/vm-entrypoint.sh &\n eend $?\n}\n\nstop() {\n ebegin \"Stopping GoClusterFS\"\n fusermount -u /mnt/goclusterfs 2>/dev/null || true\n eend $?\n}\nINITEOF", - "chmod +x /etc/init.d/goclusterfs", - "rc-update add goclusterfs default", - - # Create mount point - "mkdir -p /mnt/goclusterfs", - - # Clean up - "rm -f /root/.ash_history", - ] - } -} diff --git a/tests/vm/scripts/build-vm-image.sh b/tests/vm/scripts/build-vm-image.sh deleted file mode 100755 index f7f4c898..00000000 --- a/tests/vm/scripts/build-vm-image.sh +++ /dev/null @@ -1,217 +0,0 @@ -#!/bin/bash -# GoClusterFS VM Image Builder -# Creates an Alpine-based VM image for testing without Packer -# -# This script: -# 1. Downloads Alpine minirootfs -# 2. Creates a QCOW2 disk image -# 3. Installs Alpine to the image -# 4. Adds GoClusterFS and dependencies - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)" -RUNTIME_DIR="${SCRIPT_DIR}/../runtime" -OUTPUT_DIR="${RUNTIME_DIR}/packer-output" - -# Alpine settings -ALPINE_VERSION="3.19" -ALPINE_MIRROR="https://dl-cdn.alpinelinux.org/alpine" -MINIROOTFS_URL="${ALPINE_MIRROR}/v${ALPINE_VERSION}/releases/x86_64/alpine-minirootfs-${ALPINE_VERSION}.0-x86_64.tar.gz" - -# Image settings -IMAGE_NAME="goclusterfs-alpine.qcow2" -IMAGE_SIZE="512M" -MOUNT_POINT="/tmp/goclusterfs-vm-build" - -# Colors -RED='\033[0;31m' -GREEN='\033[0;32m' -BLUE='\033[0;34m' -NC='\033[0m' - -log_info() { echo -e "${BLUE}[INFO]${NC} $*"; } -log_success() { echo -e "${GREEN}[OK]${NC} $*"; } -log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } - -cleanup() { - log_info "Cleaning up..." - sudo umount "${MOUNT_POINT}" 2>/dev/null || true - sudo losetup -D 2>/dev/null || true - rm -rf "${MOUNT_POINT}" -} - -trap cleanup EXIT - -# Check dependencies -check_deps() { - local missing=() - for cmd in qemu-img wget tar; do - if ! command -v "$cmd" &>/dev/null; then - missing+=("$cmd") - fi - done - - if [[ ${#missing[@]} -gt 0 ]]; then - log_error "Missing dependencies: ${missing[*]}" - exit 1 - fi - - if [[ ! -f "${PROJECT_ROOT}/goclusterfs-mount" ]]; then - log_error "goclusterfs-mount binary not found. Run 'make build-mount' first." - exit 1 - fi -} - -# Download Alpine minirootfs -download_rootfs() { - local tarball="${RUNTIME_DIR}/alpine-minirootfs.tar.gz" - - if [[ -f "$tarball" ]]; then - log_info "Using cached minirootfs" - return - fi - - log_info "Downloading Alpine minirootfs..." - wget -q --show-progress -O "$tarball" "$MINIROOTFS_URL" - log_success "Downloaded minirootfs" -} - -# Create and populate the image -build_image() { - mkdir -p "${OUTPUT_DIR}" - local image_path="${OUTPUT_DIR}/${IMAGE_NAME}" - local raw_image="${OUTPUT_DIR}/temp.raw" - - if [[ -f "$image_path" ]]; then - log_info "Removing existing image..." - rm -f "$image_path" - fi - - log_info "Creating disk image..." - - # Create raw image (easier to work with) - dd if=/dev/zero of="$raw_image" bs=1M count=512 status=progress - - # Create partition and filesystem - log_info "Creating filesystem..." - mkfs.ext4 -F "$raw_image" - - # Mount and populate - mkdir -p "${MOUNT_POINT}" - sudo mount -o loop "$raw_image" "${MOUNT_POINT}" - - log_info "Extracting Alpine rootfs..." - sudo tar -xzf "${RUNTIME_DIR}/alpine-minirootfs.tar.gz" -C "${MOUNT_POINT}" - - # Configure Alpine - log_info "Configuring Alpine..." - - # Set up resolv.conf - echo "nameserver 8.8.8.8" | sudo tee "${MOUNT_POINT}/etc/resolv.conf" > /dev/null - - # Set up repositories - cat < /dev/null -${ALPINE_MIRROR}/v${ALPINE_VERSION}/main -${ALPINE_MIRROR}/v${ALPINE_VERSION}/community -EOF - - # Install packages using chroot - log_info "Installing packages..." - sudo chroot "${MOUNT_POINT}" /bin/sh -c " - apk update - apk add --no-cache openssh fuse3 util-linux procps openrc e2fsprogs attr - - # Configure SSH - ssh-keygen -A - sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config - sed -i 's/#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config - echo 'root:testpass123' | chpasswd - - # Enable services - rc-update add sshd default - rc-update add devfs sysinit - rc-update add dmesg sysinit - rc-update add mdev sysinit - - # Create mount point - mkdir -p /mnt/goclusterfs - " - - # Copy GoClusterFS binary - log_info "Installing GoClusterFS..." - sudo cp "${PROJECT_ROOT}/goclusterfs-mount" "${MOUNT_POINT}/usr/local/bin/" - sudo chmod +x "${MOUNT_POINT}/usr/local/bin/goclusterfs-mount" - - # Copy entrypoint script - sudo cp "${SCRIPT_DIR}/vm-entrypoint.sh" "${MOUNT_POINT}/usr/local/bin/" - sudo chmod +x "${MOUNT_POINT}/usr/local/bin/vm-entrypoint.sh" - - # Set up SSH authorized keys if provided - if [[ -f "${RUNTIME_DIR}/test_key.pub" ]]; then - sudo mkdir -p "${MOUNT_POINT}/root/.ssh" - sudo cp "${RUNTIME_DIR}/test_key.pub" "${MOUNT_POINT}/root/.ssh/authorized_keys" - sudo chmod 700 "${MOUNT_POINT}/root/.ssh" - sudo chmod 600 "${MOUNT_POINT}/root/.ssh/authorized_keys" - fi - - # Create GoClusterFS init script - cat <<'INITEOF' | sudo tee "${MOUNT_POINT}/etc/init.d/goclusterfs" > /dev/null -#!/sbin/openrc-run - -depend() { - need localmount - after modules -} - -start() { - ebegin "Starting GoClusterFS" - /usr/local/bin/vm-entrypoint.sh & - eend $? -} - -stop() { - ebegin "Stopping GoClusterFS" - fusermount -u /mnt/goclusterfs 2>/dev/null || true - eend $? -} -INITEOF - sudo chmod +x "${MOUNT_POINT}/etc/init.d/goclusterfs" - sudo chroot "${MOUNT_POINT}" rc-update add goclusterfs default - - # Set hostname - echo "goclusterfs-vm" | sudo tee "${MOUNT_POINT}/etc/hostname" > /dev/null - - # Configure fstab - cat < /dev/null -/dev/vda / ext4 defaults,noatime 0 1 -devtmpfs /dev devtmpfs defaults 0 0 -proc /proc proc defaults 0 0 -sysfs /sys sysfs defaults 0 0 -EOF - - # Unmount - sudo umount "${MOUNT_POINT}" - - # Convert to QCOW2 - log_info "Converting to QCOW2..." - qemu-img convert -f raw -O qcow2 "$raw_image" "$image_path" - rm -f "$raw_image" - - log_success "VM image created: ${image_path}" -} - -main() { - log_info "Building GoClusterFS VM image..." - - check_deps - mkdir -p "${RUNTIME_DIR}" "${OUTPUT_DIR}" - download_rootfs - build_image - - log_success "Build complete!" - log_info "Image location: ${OUTPUT_DIR}/${IMAGE_NAME}" -} - -main "$@" diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 4aa2f080..9c8d508c 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -54,6 +54,7 @@ run_test() { log_info "==========================================" log_info "Running test: $test_name" log_info "==========================================" + mark_vm_logs "Running test: $test_name" # Clean up test files from previous test to reclaim disk space cleanup_test_files 1 diff --git a/tests/vm/scripts/vm-entrypoint.sh b/tests/vm/scripts/vm-entrypoint.sh deleted file mode 100755 index 4327ad07..00000000 --- a/tests/vm/scripts/vm-entrypoint.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/bin/sh -# GoClusterFS VM Entrypoint Script -# Runs inside the Alpine VM to start GoClusterFS -# -# This script: -# 1. Waits for the shared block device to be available -# 2. Optionally formats the filesystem (if FORMAT_DISK=true) -# 3. Mounts GoClusterFS at /mnt/goclusterfs -# 4. Handles graceful shutdown on signals - -set -e - -# Configuration -MOUNT_POINT="/mnt/goclusterfs" -DEVICE="/dev/vdb" # Second virtio disk (shared cluster storage) -GOCLUSTERFS="/usr/local/bin/goclusterfs-mount" -LOG_FILE="/var/log/goclusterfs.log" -READY_FILE="/tmp/goclusterfs-ready" - -# Read boot parameters -VM_ID="${VM_ID:-1}" -FORMAT_DISK="${FORMAT_DISK:-false}" - -# Logging -log() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "${LOG_FILE}" -} - -log_error() { - echo "[$(date '+%Y-%m-%d %H:%M:%S')] ERROR: $*" | tee -a "${LOG_FILE}" >&2 -} - -# Wait for device -wait_for_device() { - local timeout=30 - local elapsed=0 - - log "Waiting for device ${DEVICE}..." - - while [ ! -b "${DEVICE}" ]; do - sleep 1 - elapsed=$((elapsed + 1)) - if [ $elapsed -ge $timeout ]; then - log_error "Device ${DEVICE} not found after ${timeout}s" - return 1 - fi - done - - log "Device ${DEVICE} is available" - return 0 -} - -# Signal handlers -cleanup() { - log "Received shutdown signal, unmounting..." - rm -f "${READY_FILE}" - fusermount -u "${MOUNT_POINT}" 2>/dev/null || true - log "Shutdown complete" - exit 0 -} - -trap cleanup SIGTERM SIGINT SIGQUIT - -# Main -main() { - log "GoClusterFS VM Entrypoint starting (VM_ID=${VM_ID}, FORMAT_DISK=${FORMAT_DISK})" - - # Ensure mount point exists - mkdir -p "${MOUNT_POINT}" - - # Wait for shared disk - if ! wait_for_device; then - log_error "Failed to find shared block device" - exit 1 - fi - - # Load FUSE module if needed - if [ ! -c /dev/fuse ]; then - log "Loading FUSE module..." - modprobe fuse || true - fi - - # Build mount command - MOUNT_ARGS="" - if [ "${FORMAT_DISK}" = "true" ]; then - MOUNT_ARGS="-format" - log "Will format filesystem on first mount" - fi - - log "Mounting GoClusterFS: ${GOCLUSTERFS} ${MOUNT_ARGS} ${MOUNT_POINT} ${DEVICE}" - - # Start GoClusterFS in background - ${GOCLUSTERFS} ${MOUNT_ARGS} "${MOUNT_POINT}" "${DEVICE}" >> "${LOG_FILE}" 2>&1 & - GOCLUSTERFS_PID=$! - - # Wait for mount to be ready - local timeout=30 - local elapsed=0 - while ! mountpoint -q "${MOUNT_POINT}"; do - if ! kill -0 $GOCLUSTERFS_PID 2>/dev/null; then - log_error "GoClusterFS process died unexpectedly" - cat "${LOG_FILE}" >&2 - exit 1 - fi - sleep 1 - elapsed=$((elapsed + 1)) - if [ $elapsed -ge $timeout ]; then - log_error "Mount point not ready after ${timeout}s" - exit 1 - fi - done - - log "GoClusterFS mounted successfully at ${MOUNT_POINT}" - - # Create ready marker for external health checks - touch "${READY_FILE}" - - # Wait for GoClusterFS process - wait $GOCLUSTERFS_PID -} - -main "$@" diff --git a/tests/vm/vm-test-harness.sh b/tests/vm/vm-test-harness.sh index b7c26bd1..7bd60576 100755 --- a/tests/vm/vm-test-harness.sh +++ b/tests/vm/vm-test-harness.sh @@ -11,6 +11,11 @@ # ./vm-test-harness.sh --cleanup # Clean up all resources # ./vm-test-harness.sh --start-vm 1 # Start specific VM (for debugging) # ./vm-test-harness.sh --stop-vm 1 # Stop specific VM +# +# VM N listens for SSH on 127.0.0.1:$((VM_SSH_BASE_PORT + N - 1)), default base 2222. +# A lock allows one run per checkout; runs from different checkouts on one host +# need different VM_SSH_BASE_PORT values. Each run gets its own SSH key and copy of goclusterfs-mount. VM console, QEMU and +# mount logs go to tests/vm/results/. set -euo pipefail @@ -28,28 +33,29 @@ ALPINE_MIRROR="https://dl-cdn.alpinelinux.org/alpine" ISO_NAME="alpine-virt-${ALPINE_VERSION}.0-x86_64.iso" ISO_URL="${ALPINE_MIRROR}/v${ALPINE_VERSION}/releases/x86_64/${ISO_NAME}" ISO_PATH="${VM_DIR}/${ISO_NAME}" +ISO_SHA256="95ed37b1e54b04aad1c6ff54a5fd60d904363a2a8810c3e3dfe4c2a29e94056b" # Shared storage (configurable via VM_CLUSTER_SIZE_MB environment variable) SHARED_DISK_PATH="${VM_DIR}/cluster.img" SHARED_DISK_SIZE_MB=${VM_CLUSTER_SIZE_MB:-1024} # Network settings (each VM gets sequential ports) -SSH_BASE_PORT=2222 -QEMU_MONITOR_BASE_PORT=4444 +SSH_BASE_PORT=${VM_SSH_BASE_PORT:-2222} # VM resources VM_MEMORY="512M" VM_CPUS=1 +VM_BOOT_ATTEMPTS=2 + +# QEMU dies with the shell that started it, so a killed run cannot leave VMs behind +VM_REAPER=(setpriv --pdeathsig KILL --) # SSH settings -SSH_KEY_PATH="${VM_DIR}/test_key" SSH_USER="root" -SSH_TIMEOUT=30 -SSH_OPTIONS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5" +SSH_OPTIONS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=5 -o BatchMode=yes -o IdentitiesOnly=yes" # Test settings DEFAULT_NODE_COUNT=2 -HEALTH_CHECK_RETRIES=60 HEALTH_CHECK_INTERVAL=2 # Colors for output @@ -93,21 +99,133 @@ get_cpu_type() { } # ============================================================================= -# SSH Key Management +# Run State # ============================================================================= -generate_ssh_key() { - if [[ -f "${SSH_KEY_PATH}" ]]; then +# A run owns a private directory holding its SSH key, a frozen copy of +# goclusterfs-mount and the VM setup disk. Child scripts inherit it. +init_vm_run() { + if [[ -n "${VM_RUN_DIR:-}" && -d "${VM_RUN_DIR}" ]]; then return 0 fi + if ! command -v setpriv > /dev/null; then + log_error "setpriv (util-linux) is required" + return 1 + fi + if [[ ! -x "${PROJECT_ROOT}/goclusterfs-mount" ]]; then + log_error "${PROJECT_ROOT}/goclusterfs-mount not found, run 'make build' first" + return 1 + fi - log_info "Generating SSH key for VM access..." - mkdir -p "${VM_DIR}" - ssh-keygen -t ed25519 -f "${SSH_KEY_PATH}" -N "" -q - log_success "SSH key generated: ${SSH_KEY_PATH}" + mkdir -p "${VM_DIR}" "${RESULTS_DIR}" + exec {VM_RUN_LOCK_FD}> "${VM_DIR}/harness.lock" + if ! flock -n "$VM_RUN_LOCK_FD"; then + log_error "Another VM run is using ${VM_DIR}. Wait for it, stop it with --cleanup, or run from another checkout with a different VM_SSH_BASE_PORT" + return 1 + fi + VM_RUN_DIR=$(mktemp -d "${VM_DIR}/run.XXXXXX") + VM_RUN_OWNER=$$ + export VM_RUN_DIR VM_RUN_OWNER VM_RUN_LOCK_FD + + cp "${PROJECT_ROOT}/goclusterfs-mount" "${VM_RUN_DIR}/goclusterfs-mount" + ssh-keygen -t ed25519 -f "$(get_ssh_key)" -N "" -q -C "goclusterfs-vm-run" + write_setup_disk "${VM_RUN_DIR}/setup.img" + rm -f "${RESULTS_DIR}"/vm*.log + log_info "Run directory: ${VM_RUN_DIR}, VM logs: ${RESULTS_DIR}/vm*.log" +} + +start_vm_run() { + init_vm_run + trap on_vm_run_exit EXIT + trap 'exit 130' INT + trap 'exit 143' TERM + if [[ "${VM_RUN_OWNER}" == "$$" && -n "$(vm_ids_with_pid_files)" ]]; then + log_warn "Stopping VMs left in ${VM_DIR} by an earlier run" + kill_all_vms + fi +} + +# Removes run directories unless a run in this checkout still holds the lock +remove_idle_run_dirs() { + local lock_fd + [[ -d "${VM_DIR}" ]] || return 0 + exec {lock_fd}> "${VM_DIR}/harness.lock" + if flock -n "$lock_fd"; then + rm -rf "${VM_DIR}"/run.* + else + log_warn "A VM run is active in ${VM_DIR}, keeping its run directory" + fi + exec {lock_fd}>&- +} + +on_vm_run_exit() { + local status=$? + trap - EXIT + trap '' INT TERM + set +e + stop_child_processes + kill_all_vms + if [[ "${VM_RUN_OWNER:-}" == "$$" ]] || ! kill -0 "${VM_RUN_OWNER:-$$}" 2>/dev/null; then + rm -rf "${VM_RUN_DIR}" + fi + exit "$status" } -get_ssh_public_key() { - cat "${SSH_KEY_PATH}.pub" +kill_process_tree() { + local pid="$1" + local children child + children=$(pgrep -P "$pid" || true) + kill -TERM "$pid" 2>/dev/null || true + for child in $children; do + kill_process_tree "$child" + done +} + +stop_child_processes() { + local child + for child in $(pgrep -P $$ || true); do + kill_process_tree "$child" + done + wait 2>/dev/null || true +} + +get_ssh_key() { + echo "${VM_RUN_DIR}/id_ed25519" +} + +# Runs once per boot via "sh /dev/vdb". Typing long lines on the serial console +# is unreliable because the guest shell's line editor wraps them at 80 columns. +write_setup_disk() { + local image="$1" + { + echo "repo='${ALPINE_MIRROR}/v${ALPINE_VERSION}'" + echo "key='$(cat "$(get_ssh_key).pub")'" + echo "mountpoint='${GOCLUSTERFS_MOUNT}'" + cat <<'EOF' +set -x +setup-interfaces -a +ifup eth0 +printf '%s\n' "$repo/main" "$repo/community" > /etc/apk/repositories +tries=0 +until apk add --no-cache openssh fuse3 gcompat attr; do + tries=$((tries + 1)) + [ "$tries" -lt 5 ] || exit 1 + sleep 3 +done +modprobe fuse +modprobe virtio_console +for port in /sys/class/virtio-ports/*; do + [ "$(cat "$port/name")" = goclusterfs.log ] && ln -sf "/dev/${port##*/}" /dev/goclusterfs-log +done +[ -c /dev/goclusterfs-log ] || exit 1 +mkdir -p /root/.ssh "$mountpoint" +chmod 700 /root/.ssh +echo "$key" > /root/.ssh/authorized_keys +chmod 600 /root/.ssh/authorized_keys +ssh-keygen -A +rc-service sshd start +EOF + } > "$image" + truncate -s %512 "$image" } # ============================================================================= @@ -116,13 +234,18 @@ get_ssh_public_key() { download_iso() { if [[ -f "${ISO_PATH}" ]]; then log_info "Alpine ISO already downloaded" - return 0 + else + log_info "Downloading Alpine Linux ISO..." + mkdir -p "${VM_DIR}" + wget -q --show-progress -O "${ISO_PATH}" "${ISO_URL}" + log_success "Downloaded: ${ISO_PATH}" fi - log_info "Downloading Alpine Linux ISO..." - mkdir -p "${VM_DIR}" - wget -q --show-progress -O "${ISO_PATH}" "${ISO_URL}" - log_success "Downloaded: ${ISO_PATH}" + if ! echo "${ISO_SHA256} ${ISO_PATH}" | sha256sum -c --status; then + log_error "${ISO_PATH} does not match the pinned SHA-256; removed it, rerun to download again" + rm -f "${ISO_PATH}" + return 1 + fi } # ============================================================================= @@ -165,6 +288,21 @@ get_vm_monitor_socket() { echo "${VM_DIR}/vm${vm_id}.monitor.sock" } +get_vm_serial_socket() { + local vm_id="$1" + echo "${VM_DIR}/vm${vm_id}.sock" +} + +get_vm_console_log() { + local vm_id="$1" + echo "${RESULTS_DIR}/vm${vm_id}.console.log" +} + +get_vm_mount_log() { + local vm_id="$1" + echo "${RESULTS_DIR}/vm${vm_id}.mount.log" +} + get_vm_ip() { local vm_id="$1" # VMs use user-mode networking with port forwarding @@ -175,54 +313,80 @@ is_vm_running() { local vm_id="$1" local pid_file=$(get_vm_pid_file "$vm_id") + # A stale pid file may name a reused PID, so check that it is still this VM's QEMU if [[ -f "$pid_file" ]]; then local pid=$(cat "$pid_file") - if kill -0 "$pid" 2>/dev/null; then + local cmdline=$(tr '\0' ' ' < "/proc/${pid}/cmdline" 2>/dev/null) + if [[ "$cmdline" == *"-name goclusterfs-vm${vm_id} "* ]]; then return 0 fi fi return 1 } -# Start a VM with Alpine live ISO +mark_vm_log() { + local vm_id="$1" + local text="$2" + [[ -d "${RESULTS_DIR}" ]] || return 0 + echo "=== $(date '+%F %T') $text" >> "$(get_vm_mount_log "$vm_id")" +} + +# Marks every VM mount log, e.g. with the name of the test about to run +mark_vm_logs() { + local log + for log in "${RESULTS_DIR}"/vm*.mount.log; do + if [[ -f "$log" ]]; then + echo "=== $(date '+%F %T') $*" >> "$log" + fi + done +} + +# Start a VM, retrying the whole bring-up once if it fails start_vm() { local vm_id="$1" - local format_disk="${2:-false}" # Whether this VM should format the shared disk + local attempt + + if is_vm_running "$vm_id"; then + log_info "VM $vm_id is already running" + return 0 + fi + download_iso || return 1 + create_shared_disk || return 1 + + for attempt in $(seq 1 "$VM_BOOT_ATTEMPTS"); do + if boot_vm "$vm_id"; then + return 0 + fi + log_warn "VM $vm_id bring-up failed (attempt $attempt of $VM_BOOT_ATTEMPTS)" + kill_vm "$vm_id" + done + log_error "VM $vm_id failed to start" + return 1 +} + +# Boot the Alpine live ISO, log in on the serial console and run the setup disk +boot_vm() { + local vm_id="$1" local ssh_port=$(get_vm_ssh_port "$vm_id") local monitor_socket=$(get_vm_monitor_socket "$vm_id") - local serial_socket="${VM_DIR}/vm${vm_id}.sock" + local serial_socket=$(get_vm_serial_socket "$vm_id") + local console_log=$(get_vm_console_log "$vm_id") local pid_file=$(get_vm_pid_file "$vm_id") - local log_file="${VM_DIR}/vm${vm_id}.log" local accelerator=$(get_accelerator) local cpu_type=$(get_cpu_type) - if is_vm_running "$vm_id"; then - log_info "VM $vm_id is already running" - return 0 - fi - # Clean up old sockets before starting (avoids "Address already in use") rm -f "$monitor_socket" "$serial_socket" - # Ensure prerequisites - download_iso - create_shared_disk - generate_ssh_key - - log_info "Starting VM $vm_id (SSH port: $ssh_port, format: $format_disk)..." + log_info "Starting VM $vm_id (SSH port: $ssh_port)..." log_info "Using accelerator: $accelerator, cpu: $cpu_type" - # Create cloud-init style seed for automatic configuration - local seed_dir="${VM_DIR}/seed${vm_id}" - mkdir -p "$seed_dir" - - # Create startup script that will be passed via kernel command line - local ssh_pubkey=$(get_ssh_public_key) + echo "=== $(date '+%F %T') VM $vm_id boot" >> "$console_log" + mark_vm_log "$vm_id" "VM $vm_id boot" + local console_offset=$(stat -c%s "$console_log") - # Start QEMU with Alpine live ISO - # The ISO boots to a login prompt - we configure it via serial console - qemu-system-x86_64 \ + "${VM_REAPER[@]}" qemu-system-x86_64 \ -name "goclusterfs-vm${vm_id}" \ -machine q35,accel=${accelerator} \ -cpu ${cpu_type} \ @@ -231,116 +395,109 @@ start_vm() { -cdrom "${ISO_PATH}" \ -boot d \ -drive "file=${SHARED_DISK_PATH},format=raw,if=virtio,cache=none,file.locking=off" \ - -netdev "user,id=net0,hostfwd=tcp::${ssh_port}-:22" \ + -drive "file=${VM_RUN_DIR}/setup.img,format=raw,if=virtio,readonly=on" \ + -netdev "user,id=net0,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ -device "virtio-net-pci,netdev=net0" \ + -device virtio-serial-pci \ + -chardev "file,id=mountlog,path=$(get_vm_mount_log "$vm_id"),append=on" \ + -device "virtserialport,chardev=mountlog,name=goclusterfs.log" \ + -chardev "socket,id=serial0,path=${serial_socket},server=on,wait=off,logfile=${console_log},logappend=on" \ + -serial chardev:serial0 \ -monitor "unix:${monitor_socket},server,nowait" \ - -serial "unix:${serial_socket},server,nowait" \ -display none \ - -daemonize \ - -pidfile "${pid_file}" \ - 2>&1 || { - log_error "Failed to start VM $vm_id" - return 1 - } + {VM_RUN_LOCK_FD}>&- >> "${RESULTS_DIR}/vm${vm_id}.qemu.log" 2>&1 & + local pid=$! + disown "$pid" + echo "$pid" > "$pid_file" + log_info "VM $vm_id started (PID: $pid)" + + if ! wait_for_console "$vm_id" "$console_offset" "login:" 120; then + report_boot_failure "$vm_id" "$console_offset" "no login prompt on the serial console" + return 1 + fi + if ! console_login "$vm_id" "$console_offset"; then + report_boot_failure "$vm_id" "$console_offset" "serial console login failed" + return 1 + fi - log_info "VM $vm_id started (PID: $(cat "$pid_file"))" + log_info "Running setup disk on VM $vm_id..." + console_send "$vm_id" "sh /dev/vdb" + if ! wait_for_ssh "$vm_id" 180; then + report_boot_failure "$vm_id" "$console_offset" "SSH not reachable after setup" + return 1 + fi - # Wait for VM to boot and configure it - sleep 10 - configure_vm "$vm_id" "$format_disk" + log_success "VM $vm_id configured and SSH verified" } -# Configure a running VM via serial console after boot -configure_vm() { +console_output_since() { local vm_id="$1" - local format_disk="${2:-false}" + local offset="$2" + tail -c +"$((offset + 1))" "$(get_vm_console_log "$vm_id")" 2>/dev/null || true +} - log_info "Configuring VM $vm_id via serial console..." +console_send() { + local vm_id="$1" + local line="$2" + printf '%s\n' "$line" | nc -U -q 1 "$(get_vm_serial_socket "$vm_id")" > /dev/null 2>&1 || true +} + +wait_for_console() { + local vm_id="$1" + local offset="$2" + local text="$3" + local timeout="$4" + local elapsed=0 - # Always configure via serial - QEMU port forwarding makes port appear open - # even before sshd is running, so we can't use nc -z to detect readiness - configure_vm_via_serial "$vm_id" "$format_disk" + until [[ "$(console_output_since "$vm_id" "$offset")" == *"$text"* ]]; do + if (( elapsed >= timeout )) || ! is_vm_running "$vm_id"; then + return 1 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done } -# Configure VM by sending commands through serial socket -configure_vm_via_serial() { +console_login() { local vm_id="$1" - local format_disk="$2" - local serial_sock="${VM_DIR}/vm${vm_id}.sock" - local ssh_pubkey=$(get_ssh_public_key) - - # Helper to send a command via serial socket - send_serial_cmd() { - local cmd="$1" - local wait="${2:-1}" - # Send command with newline - echo "$cmd" | nc -U -w 1 "$serial_sock" 2>/dev/null || true - sleep "$wait" - } - - # Wait for login prompt - log_info "Waiting for VM $vm_id login prompt..." - sleep 15 # Wait for boot - - # Login as root (no password on Alpine live) - log_info "Logging into VM $vm_id..." - send_serial_cmd "root" 2 - - # Configure networking - log_info "Configuring networking..." - send_serial_cmd "setup-interfaces -a" 3 - send_serial_cmd "ifup eth0" 2 - - # Configure Alpine repositories (live ISO only has CD-ROM packages) - log_info "Configuring Alpine repositories..." - send_serial_cmd "echo 'https://dl-cdn.alpinelinux.org/alpine/v3.19/main' > /etc/apk/repositories" 1 - send_serial_cmd "echo 'https://dl-cdn.alpinelinux.org/alpine/v3.19/community' >> /etc/apk/repositories" 1 - send_serial_cmd "apk update" 10 - - # Install and configure SSH - log_info "Installing SSH..." - send_serial_cmd "apk add --no-cache openssh" 15 - - log_info "Configuring SSH..." - send_serial_cmd "sed -i 's/#PermitRootLogin.*/PermitRootLogin yes/' /etc/ssh/sshd_config" 1 - send_serial_cmd "sed -i 's/#PasswordAuthentication.*/PasswordAuthentication yes/' /etc/ssh/sshd_config" 1 - send_serial_cmd "ssh-keygen -A" 3 - send_serial_cmd "rc-service sshd start" 3 - - # Set root password - log_info "Setting root password..." - send_serial_cmd "echo 'root:testpass123' | chpasswd" 1 - - # Set up SSH key - log_info "Installing SSH key..." - send_serial_cmd "mkdir -p /root/.ssh" 1 - send_serial_cmd "chmod 700 /root/.ssh" 1 - send_serial_cmd "echo '$ssh_pubkey' > /root/.ssh/authorized_keys" 1 - send_serial_cmd "chmod 600 /root/.ssh/authorized_keys" 1 - - # Verify SSH is actually working before returning - # This prevents race conditions where start_vm returns before SSH is ready - log_info "Verifying SSH connectivity for VM $vm_id..." - local ssh_port=$(get_vm_ssh_port "$vm_id") - local ssh_timeout=60 - local ssh_elapsed=0 + local offset="$2" + local attempt - while [[ $ssh_elapsed -lt $ssh_timeout ]]; do - if sshpass -p "testpass123" ssh ${SSH_OPTIONS} -p "$ssh_port" "${SSH_USER}@127.0.0.1" "true" 2>/dev/null; then - log_success "VM $vm_id configured and SSH verified" + for attempt in 1 2 3; do + console_send "$vm_id" "root" + if wait_for_console "$vm_id" "$offset" ":~# " 15; then return 0 fi - sleep 2 - ssh_elapsed=$((ssh_elapsed + 2)) done - - log_error "VM $vm_id: SSH verification failed after ${ssh_timeout}s" return 1 } -# Alias for backward compatibility -configure_vm_via_monitor() { - configure_vm_via_serial "$@" +wait_for_ssh() { + local vm_id="$1" + local timeout="$2" + local elapsed=0 + + until ssh_run "$vm_id" true < /dev/null 2>/dev/null; do + if (( elapsed >= timeout )) || ! is_vm_running "$vm_id"; then + return 1 + fi + sleep 2 + elapsed=$((elapsed + 2)) + done +} + +report_boot_failure() { + local vm_id="$1" + local offset="$2" + local reason="$3" + + log_error "VM $vm_id: $reason" + if ! is_vm_running "$vm_id"; then + log_error "QEMU exited, see ${RESULTS_DIR}/vm${vm_id}.qemu.log:" + tail -n 5 "${RESULTS_DIR}/vm${vm_id}.qemu.log" >&2 || true + fi + log_error "Console output (full log: $(get_vm_console_log "$vm_id")):" + console_output_since "$vm_id" "$offset" | tail -n 20 >&2 } # ============================================================================= @@ -355,33 +512,22 @@ setup_goclusterfs() { log_info "Setting up GoClusterFS on VM $vm_id (format=$format_disk)..." - # Copy the binary to the VM log_info "Copying goclusterfs-mount binary..." - sshpass -p "testpass123" scp ${SSH_OPTIONS} -P "$ssh_port" \ - "${PROJECT_ROOT}/goclusterfs-mount" \ - "root@127.0.0.1:/usr/local/bin/goclusterfs-mount" - - # Install fuse3, gcompat (glibc compatibility for binary compiled on glibc systems) and attr (setfattr for snapshot tests) - log_info "Installing fuse3, gcompat and attr..." - ssh_run_password "$vm_id" "apk add --no-cache fuse3 gcompat attr" - - # Load FUSE kernel module (required before mounting any FUSE filesystem) - log_info "Loading FUSE kernel module..." - ssh_run_password "$vm_id" "modprobe fuse" - - # Create mount point - ssh_run_password "$vm_id" "mkdir -p /mnt/goclusterfs" + scp ${SSH_OPTIONS} -i "$(get_ssh_key)" -P "$ssh_port" \ + "${VM_RUN_DIR}/goclusterfs-mount" \ + "${SSH_USER}@127.0.0.1:/usr/local/bin/goclusterfs-mount" # Mount GoClusterFS - local mount_args="-mount /mnt/goclusterfs -device /dev/vda" + local mount_args="-mount ${GOCLUSTERFS_MOUNT} -device /dev/vda" if [[ "$format_disk" == "true" ]]; then log_info "Formatting shared disk on VM $vm_id..." mount_args="$mount_args -format" fi + # /var/log/goclusterfs.log is lost when the VM dies; the log device keeps a copy on the host log_info "Mounting GoClusterFS..." - ssh_run_password "$vm_id" \ - "nohup /usr/local/bin/goclusterfs-mount $mount_args > /var/log/goclusterfs.log 2>&1 &" + ssh_run "$vm_id" \ + "nohup /usr/local/bin/goclusterfs-mount $mount_args < /dev/null 2>&1 | tee /var/log/goclusterfs.log > /dev/goclusterfs-log 2>/dev/null &" # Wait for mount to be ready wait_for_goclusterfs_mount "$vm_id" @@ -396,7 +542,7 @@ wait_for_goclusterfs_mount() { log_info "Waiting for GoClusterFS mount on VM $vm_id..." while [[ $elapsed -lt $timeout ]]; do - if ssh_run_password "$vm_id" "mountpoint -q /mnt/goclusterfs" 2>/dev/null; then + if ssh_run "$vm_id" "mountpoint -q ${GOCLUSTERFS_MOUNT}" 2>/dev/null; then log_success "GoClusterFS mounted on VM $vm_id" return 0 fi @@ -404,8 +550,8 @@ wait_for_goclusterfs_mount() { elapsed=$((elapsed + 1)) done - log_error "GoClusterFS mount timeout on VM $vm_id" - ssh_run_password "$vm_id" "cat /var/log/goclusterfs.log" 2>/dev/null || true + log_error "GoClusterFS mount timeout on VM $vm_id, last lines of $(get_vm_mount_log "$vm_id"):" + tail -n 20 "$(get_vm_mount_log "$vm_id")" >&2 || true return 1 } @@ -416,14 +562,16 @@ stop_vm() { if ! is_vm_running "$vm_id"; then log_info "VM $vm_id is not running" + rm -f "$pid_file" "$(get_vm_monitor_socket "$vm_id")" "$(get_vm_serial_socket "$vm_id")" return 0 fi log_info "Stopping VM $vm_id gracefully..." + mark_vm_log "$vm_id" "VM $vm_id stop" # Try graceful shutdown via monitor (Unix socket) local monitor_socket=$(get_vm_monitor_socket "$vm_id") - echo "system_powerdown" | nc -U -q1 "$monitor_socket" 2>/dev/null || true + echo "system_powerdown" | nc -U -q1 "$monitor_socket" > /dev/null 2>&1 || true # Wait for graceful shutdown local timeout=30 @@ -440,7 +588,7 @@ stop_vm() { rm -f "$pid_file" # Clean up sockets - rm -f "$(get_vm_monitor_socket "$vm_id")" "${VM_DIR}/vm${vm_id}.sock" + rm -f "$(get_vm_monitor_socket "$vm_id")" "$(get_vm_serial_socket "$vm_id")" log_success "VM $vm_id stopped" } @@ -449,17 +597,33 @@ kill_vm() { local vm_id="$1" local pid_file=$(get_vm_pid_file "$vm_id") - if [[ -f "$pid_file" ]]; then - local pid=$(cat "$pid_file") - kill -9 "$pid" 2>/dev/null || true - rm -f "$pid_file" + if is_vm_running "$vm_id"; then + kill -9 "$(cat "$pid_file")" 2>/dev/null || true + mark_vm_log "$vm_id" "VM $vm_id killed" fi + rm -f "$pid_file" # Clean up sockets - rm -f "$(get_vm_monitor_socket "$vm_id")" "${VM_DIR}/vm${vm_id}.sock" + rm -f "$(get_vm_monitor_socket "$vm_id")" "$(get_vm_serial_socket "$vm_id")" log_info "VM $vm_id killed" } +vm_ids_with_pid_files() { + local pid_file + for pid_file in "${VM_DIR}"/vm*.pid; do + if [[ -f "$pid_file" ]]; then + basename "$pid_file" | sed 's/vm\([0-9]*\).pid/\1/' + fi + done +} + +kill_all_vms() { + local vm_id + for vm_id in $(vm_ids_with_pid_files); do + kill_vm "$vm_id" + done +} + # ============================================================================= # SSH Functions for Tests # ============================================================================= @@ -468,28 +632,24 @@ ssh_run() { shift local ssh_port=$(get_vm_ssh_port "$vm_id") - ssh ${SSH_OPTIONS} -i "${SSH_KEY_PATH}" -p "$ssh_port" "${SSH_USER}@127.0.0.1" "$@" + ssh ${SSH_OPTIONS} -i "$(get_ssh_key)" -p "$ssh_port" "${SSH_USER}@127.0.0.1" "$@" } +# VMs accept only the run's SSH key; kept for existing callers ssh_run_password() { - local vm_id="$1" - shift - local ssh_port=$(get_vm_ssh_port "$vm_id") - - sshpass -p "testpass123" ssh ${SSH_OPTIONS} -p "$ssh_port" "${SSH_USER}@127.0.0.1" "$@" + ssh_run "$@" } wait_vm_healthy() { local vm_id="$1" local timeout="${2:-120}" - local ssh_port=$(get_vm_ssh_port "$vm_id") log_info "Waiting for VM $vm_id to be healthy..." local elapsed=0 while [[ $elapsed -lt $timeout ]]; do # First check if SSH is reachable at all - if sshpass -p "testpass123" ssh ${SSH_OPTIONS} -p "$ssh_port" "${SSH_USER}@127.0.0.1" "true" 2>/dev/null; then + if ssh_run "$vm_id" "true" 2>/dev/null; then log_success "VM $vm_id is healthy (SSH reachable)" return 0 fi @@ -508,12 +668,11 @@ wait_vm_healthy() { # restarts the VM and waits for it to become healthy. ensure_vm_running() { local vm_id="$1" - local ssh_port=$(get_vm_ssh_port "$vm_id") # Check if VM is reachable via SSH - if sshpass -p "testpass123" ssh ${SSH_OPTIONS} -p "$ssh_port" "${SSH_USER}@127.0.0.1" "true" 2>/dev/null; then + if ssh_run "$vm_id" "true" 2>/dev/null; then # VM is healthy, check if GoClusterFS is mounted - if sshpass -p "testpass123" ssh ${SSH_OPTIONS} -p "$ssh_port" "${SSH_USER}@127.0.0.1" "mountpoint -q /mnt/goclusterfs" 2>/dev/null; then + if ssh_run "$vm_id" "mountpoint -q ${GOCLUSTERFS_MOUNT}" 2>/dev/null; then return 0 # Fully healthy fi # SSH works but GoClusterFS not mounted - remount @@ -557,21 +716,21 @@ write_file_on_vm() { local filename="$2" local content="$3" - ssh_run_password "$vm_id" "echo '$content' > ${GOCLUSTERFS_MOUNT}/$filename" + ssh_run "$vm_id" "echo '$content' > ${GOCLUSTERFS_MOUNT}/$filename" } read_file_on_vm() { local vm_id="$1" local filename="$2" - ssh_run_password "$vm_id" "cat ${GOCLUSTERFS_MOUNT}/$filename" + ssh_run "$vm_id" "cat ${GOCLUSTERFS_MOUNT}/$filename" } file_exists_on_vm() { local vm_id="$1" local filename="$2" - ssh_run_password "$vm_id" "test -f ${GOCLUSTERFS_MOUNT}/$filename" + ssh_run "$vm_id" "test -f ${GOCLUSTERFS_MOUNT}/$filename" } # Clean up test files from the GoClusterFS filesystem @@ -583,7 +742,7 @@ cleanup_test_files() { log_info "Cleaning up test files from GoClusterFS..." # Check if VM is reachable and GoClusterFS is mounted - if ! ssh_run_password "$vm_id" "mountpoint -q ${GOCLUSTERFS_MOUNT}" 2>/dev/null; then + if ! ssh_run "$vm_id" "mountpoint -q ${GOCLUSTERFS_MOUNT}" 2>/dev/null; then log_warn "GoClusterFS not mounted on VM $vm_id, skipping cleanup" return 0 fi @@ -591,14 +750,14 @@ cleanup_test_files() { # Remove all files and directories from mount point # Use find to avoid "argument list too long" errors with many files # Also handles hidden files (.*) safely - ssh_run_password "$vm_id" "find ${GOCLUSTERFS_MOUNT} -mindepth 1 -delete 2>/dev/null || rm -rf ${GOCLUSTERFS_MOUNT}/* ${GOCLUSTERFS_MOUNT}/.[!.]* 2>/dev/null" || true + ssh_run "$vm_id" "find ${GOCLUSTERFS_MOUNT} -mindepth 1 -delete 2>/dev/null || rm -rf ${GOCLUSTERFS_MOUNT}/* ${GOCLUSTERFS_MOUNT}/.[!.]* 2>/dev/null" || true # Sync to ensure blocks are freed - ssh_run_password "$vm_id" "sync" + ssh_run "$vm_id" "sync" # Log free space for debugging local free_space - free_space=$(ssh_run_password "$vm_id" "df -h ${GOCLUSTERFS_MOUNT} 2>/dev/null | tail -1 | awk '{print \$4}'" 2>/dev/null) || true + free_space=$(ssh_run "$vm_id" "df -h ${GOCLUSTERFS_MOUNT} 2>/dev/null | tail -1 | awk '{print \$4}'" 2>/dev/null) || true if [[ -n "$free_space" ]]; then log_info "Free space after cleanup: $free_space" fi @@ -612,12 +771,9 @@ cleanup_test_files() { cleanup() { log_info "Cleaning up all VMs..." - # Stop all VMs - for pid_file in "${VM_DIR}"/vm*.pid; do - if [[ -f "$pid_file" ]]; then - local vm_id=$(basename "$pid_file" | sed 's/vm\([0-9]*\).pid/\1/') - stop_vm "$vm_id" 2>/dev/null || true - fi + local vm_id + for vm_id in $(vm_ids_with_pid_files); do + stop_vm "$vm_id" 2>/dev/null || true done log_success "Cleanup complete" @@ -640,8 +796,9 @@ run_tests() { log_info "Running VM test suite with $node_count nodes..." - # Run the test script - "${SCRIPT_DIR}/scripts/run-vm-tests.sh" "$node_count" + # In the background so traps run while waiting; the runner gets SIGTERM if this shell dies + setpriv --pdeathsig TERM -- "${SCRIPT_DIR}/scripts/run-vm-tests.sh" "$node_count" & + wait $! } # ============================================================================= @@ -655,15 +812,20 @@ main() { shift local nodes="${1:-$DEFAULT_NODE_COUNT}" [[ "$nodes" == --nodes=* ]] && nodes="${nodes#--nodes=}" + start_vm_run run_tests "$nodes" ;; --nodes=*) local nodes="${command#--nodes=}" + start_vm_run run_tests "$nodes" ;; --start-vm) shift - start_vm "$1" "${2:-false}" + VM_REAPER=(setsid) + init_vm_run + start_vm "$1" + log_info "SSH: ssh -i $(get_ssh_key) -p $(get_vm_ssh_port "$1") root@127.0.0.1" ;; --stop-vm) shift @@ -675,6 +837,7 @@ main() { ;; --cleanup) cleanup + remove_idle_run_dirs ;; --cleanup-all) cleanup_all @@ -684,12 +847,16 @@ main() { echo "" echo "Usage:" echo " $0 --run-tests [--nodes=N] Run tests with N VMs (default: 2)" - echo " $0 --start-vm ID [format] Start specific VM" + echo " $0 --start-vm ID Start specific VM" echo " $0 --stop-vm ID Stop specific VM" echo " $0 --kill-vm ID Force kill VM" echo " $0 --cleanup Stop all VMs" echo " $0 --cleanup-all Remove all runtime files" echo " $0 --help Show this help" + echo "" + echo "Environment:" + echo " VM_SSH_BASE_PORT Host SSH port of VM 1 (default: 2222); use a different" + echo " value for each checkout that runs VMs at the same time" ;; *) log_error "Unknown command: $command" @@ -699,7 +866,9 @@ main() { esac } -# Run if executed directly +# Run if executed directly; a sourcing test runner joins the run or starts its own if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then main "$@" +else + start_vm_run fi