diff --git a/tests/docker/scripts/run-tests.sh b/tests/docker/scripts/run-tests.sh index 2a5fbd0b..d3f2ec94 100644 --- a/tests/docker/scripts/run-tests.sh +++ b/tests/docker/scripts/run-tests.sh @@ -83,11 +83,19 @@ list_dir() { exec_on_node "$node" ls -la "/mnt/goclusterfs/$path" } -# Kill a node container +# Kill a node container, as a crash would, and fail unless it stops within 10s kill_node() { - local node=$1 + local node=$1 waited=0 log "Killing node: $node" - docker kill "$node" || true + docker kill "$node" >/dev/null 2>&1 || true + while [[ "$(docker inspect -f '{{.State.Running}}' "$node" 2>/dev/null)" == true ]]; do + if [[ $waited -ge 50 ]]; then + log "ERROR: $node still running 10s after docker kill" + return 1 + fi + sleep 0.2 + waited=$((waited + 1)) + done } # Restart a node container @@ -1189,7 +1197,7 @@ test_journal_recovery_multinode() { # Phase 2: Kill Node A to simulate crash log "Phase 2: Killing Node A to simulate crash..." - kill_node "$node_a" + kill_node "$node_a" || return 1 sleep 2 # Phase 3: Wait for heartbeat to become stale @@ -1272,7 +1280,10 @@ test_restart_keeps_peer_journal() { local since slot_before since=$(date -u +%Y-%m-%dT%H:%M:%SZ) slot_before=$(docker logs "$node_a" 2>&1 | grep -o 'Registered as node [0-9]*' | tail -1) - docker kill "$node_a" >/dev/null 2>&1 || true + if ! kill_node "$node_a"; then + exec_on_node "$node_b" touch /tmp/peer-writer.stop + return 1 + fi sleep 1 docker start "$node_a" >/dev/null if ! wait_node_healthy "$node_a" 60; then @@ -1357,7 +1368,7 @@ test_journal_recovery_multiple_files() { # Phase 2: Kill Node A to simulate crash log "Phase 2: Killing Node A to simulate crash..." - kill_node "$node_a" + kill_node "$node_a" || return 1 sleep 2 # Phase 3: Wait for heartbeat to become stale @@ -1447,7 +1458,7 @@ test_journal_recovery_large_file() { # Phase 2: Kill Node A to simulate crash log "Phase 2: Killing Node A to simulate crash..." - kill_node "$node_a" + kill_node "$node_a" || return 1 sleep 2 # Phase 3: Wait for heartbeat to become stale @@ -1521,7 +1532,7 @@ test_journal_recovery_idempotency() { # Phase 2: Kill Node A log "Phase 2: Killing Node A..." - kill_node "$node_a" + kill_node "$node_a" || return 1 sleep 2 # Phase 3: Wait for heartbeat timeout @@ -1547,7 +1558,7 @@ test_journal_recovery_idempotency() { # Phase 5: Stop Node B log "Phase 5: Stopping Node B..." - kill_node "$node_b" + kill_node "$node_b" || return 1 sleep 2 # Phase 6: Verify idempotency on Node C @@ -1617,7 +1628,7 @@ test_journal_crash_recovery_file() { sleep 1 log "Phase 2: Simulating crash by killing container..." - kill_node "$node_a" + kill_node "$node_a" || return 1 sleep 2 # Phase 3: Restart (remount) - this triggers journal recovery @@ -1684,7 +1695,7 @@ test_journal_crash_recovery_directory() { sleep 1 log "Phase 2: Simulating crash by killing container..." - kill_node "$node_a" + kill_node "$node_a" || return 1 sleep 2 # Phase 3: Restart (remount) - this triggers journal recovery @@ -1764,7 +1775,7 @@ test_journal_crash_recovery_multiple_ops() { log "All operations complete, simulating crash..." # Kill the container to simulate crash - docker kill "$node_a" >/dev/null 2>&1 || true + kill_node "$node_a" || return 1 sleep 2 # Restart container @@ -1855,7 +1866,7 @@ test_journal_crash_during_write() { log "Wrote $write_count rapid files, crashing now..." # Kill container while writes may be in progress - docker kill "$node_a" >/dev/null 2>&1 || true + kill_node "$node_a" || return 1 # Wait for any background processes to terminate wait 2>/dev/null || true @@ -1973,7 +1984,7 @@ test_container_kill_recovery() { write_file "$victim" "before_kill.txt" "Written before kill" # Kill the victim - kill_node "$victim" + kill_node "$victim" || return 1 sleep 2 # Survivor should still be able to read the file @@ -2530,7 +2541,7 @@ test_stress_random_failures() { if [ $((RANDOM % 5)) -eq 0 ]; then local kill_idx=$((RANDOM % NODE_COUNT)) local kill_node="${NODE_ARRAY[$kill_idx]}" - kill_node "$kill_node" + kill_node "$kill_node" || return 1 sleep 1 restart_node "$kill_node" wait_node_healthy "$kill_node" 30 diff --git a/tests/integration/helpers.go b/tests/integration/helpers.go index 2b328e08..6858e9f1 100644 --- a/tests/integration/helpers.go +++ b/tests/integration/helpers.go @@ -522,12 +522,25 @@ func (m *MountedFS) PID() int { return 0 } -// Kill sends SIGKILL to the mount process to simulate a crash. +// Kill sends SIGKILL to the mount process to simulate a crash and waits until it is gone. func (m *MountedFS) Kill() error { if m.MountCmd == nil || m.MountCmd.Process == nil { return fmt.Errorf("mount process not running") } - return m.MountCmd.Process.Kill() + if err := m.MountCmd.Process.Kill(); err != nil { + return err + } + exited := make(chan struct{}) + go func() { + _ = m.MountCmd.Wait() + close(exited) + }() + select { + case <-exited: + return nil + case <-time.After(10 * time.Second): + return fmt.Errorf("mount process %d still running 10s after SIGKILL", m.MountCmd.Process.Pid) + } } // OpenFile opens a file in the mounted filesystem for read/write operations. diff --git a/tests/integration/multinode_helpers.go b/tests/integration/multinode_helpers.go index 8bb604de..adf48ade 100644 --- a/tests/integration/multinode_helpers.go +++ b/tests/integration/multinode_helpers.go @@ -123,9 +123,6 @@ func (c *MultiNodeCluster) KillNode(idx int) error { return err } - // Wait briefly for process to die and unmount to propagate - time.Sleep(500 * time.Millisecond) - // Force unmount to clean up mount point c.Nodes[idx].Unmount() diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 7f20bfc5..6d0e0459 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -609,7 +609,11 @@ test_restart_keeps_peer_journal() { local slot_before slot_before=$(ssh_run "$vm_a" "grep -o 'Registered as node [0-9]*' /var/log/goclusterfs.log | tail -1") - ssh_run "$vm_a" "pkill -9 goclusterfs-mount; sleep 1; umount -l /mnt/goclusterfs 2>/dev/null; true" + if ! crash_goclusterfs "$vm_a"; then + ssh_run "$vm_b" "touch /tmp/peer-writer.stop" + return 1 + fi + ssh_run "$vm_a" "umount -l /mnt/goclusterfs 2>/dev/null; true" if ! setup_goclusterfs "$vm_a" false; then ssh_run "$vm_b" "touch /tmp/peer-writer.stop" return 1 @@ -2946,13 +2950,38 @@ test_concurrent_writes_power_failure() { return 0 } +# Print the PIDs of goclusterfs-mount on VM $1: processes with an argument equal to the path setup_goclusterfs starts +# it with. On Alpine the gcompat loader runs it, so its name and argv[0] are the loader's. A zombie has no arguments. +mount_pids() { + ssh_run "$1" 'for p in /proc/[0-9]*; do tr "\0" "\n" 2>/dev/null <$p/cmdline | grep -qx "/usr/local/bin/goclusterfs-moun[t]" && echo ${p#/proc/}; done; true' +} + +# Kill goclusterfs-mount on VM $1 with SIGKILL, as a crash would, and fail unless it is gone within 10s. +crash_goclusterfs() { + local vm_id=$1 pids waited=0 + pids=$(mount_pids "$vm_id") + if [[ -z "$pids" ]]; then + log_error "VM $vm_id: no goclusterfs-mount process to kill" + return 1 + fi + ssh_run "$vm_id" "kill -9 $(echo $pids)" + while [[ -n "$(mount_pids "$vm_id")" ]]; do + if [[ $waited -ge 50 ]]; then + log_error "VM $vm_id: goclusterfs-mount still running 10s after SIGKILL" + return 1 + fi + sleep 0.2 + waited=$((waited + 1)) + done +} + # Unmount GoClusterFS on VM $1 and wait for the mount process to exit, which # marks its journal clean. unmount_goclusterfs() { local vm_id=$1 ssh_run "$vm_id" "fusermount3 -u ${GOCLUSTERFS_MOUNT}" || return 1 local waited=0 - while ssh_run "$vm_id" "pidof goclusterfs-mount > /dev/null"; do + while [[ -n "$(mount_pids "$vm_id")" ]]; do if [[ $waited -ge 30 ]]; then return 1 fi diff --git a/tests/vm/vm-test-harness.sh b/tests/vm/vm-test-harness.sh index 6ba986e5..b3a881ca 100755 --- a/tests/vm/vm-test-harness.sh +++ b/tests/vm/vm-test-harness.sh @@ -607,6 +607,15 @@ kill_vm() { if is_vm_running "$vm_id"; then kill -9 "$(cat "$pid_file")" 2>/dev/null || true + local waited=0 + while is_vm_running "$vm_id"; do + if [[ $waited -ge 50 ]]; then + log_error "VM $vm_id: QEMU still running 10s after SIGKILL" + return 1 + fi + sleep 0.2 + waited=$((waited + 1)) + done mark_vm_log "$vm_id" "VM $vm_id killed" fi rm -f "$pid_file"