Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 26 additions & 15 deletions tests/docker/scripts/run-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,19 @@ list_dir() {
exec_on_node "$node" ls -la "/mnt/goclusterfs/$path"
}

# Kill a node container
# Kill a node container, as a crash would, and fail unless it stops within 10s
kill_node() {
local node=$1
local node=$1 waited=0
log "Killing node: $node"
docker kill "$node" || true
docker kill "$node" >/dev/null 2>&1 || true
while [[ "$(docker inspect -f '{{.State.Running}}' "$node" 2>/dev/null)" == true ]]; do
if [[ $waited -ge 50 ]]; then
log "ERROR: $node still running 10s after docker kill"
return 1
fi
sleep 0.2
waited=$((waited + 1))
done
}

# Restart a node container
Expand Down Expand Up @@ -1189,7 +1197,7 @@ test_journal_recovery_multinode() {

# Phase 2: Kill Node A to simulate crash
log "Phase 2: Killing Node A to simulate crash..."
kill_node "$node_a"
kill_node "$node_a" || return 1
sleep 2

# Phase 3: Wait for heartbeat to become stale
Expand Down Expand Up @@ -1272,7 +1280,10 @@ test_restart_keeps_peer_journal() {
local since slot_before
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
slot_before=$(docker logs "$node_a" 2>&1 | grep -o 'Registered as node [0-9]*' | tail -1)
docker kill "$node_a" >/dev/null 2>&1 || true
if ! kill_node "$node_a"; then
exec_on_node "$node_b" touch /tmp/peer-writer.stop
return 1
fi
sleep 1
docker start "$node_a" >/dev/null
if ! wait_node_healthy "$node_a" 60; then
Expand Down Expand Up @@ -1357,7 +1368,7 @@ test_journal_recovery_multiple_files() {

# Phase 2: Kill Node A to simulate crash
log "Phase 2: Killing Node A to simulate crash..."
kill_node "$node_a"
kill_node "$node_a" || return 1
sleep 2

# Phase 3: Wait for heartbeat to become stale
Expand Down Expand Up @@ -1447,7 +1458,7 @@ test_journal_recovery_large_file() {

# Phase 2: Kill Node A to simulate crash
log "Phase 2: Killing Node A to simulate crash..."
kill_node "$node_a"
kill_node "$node_a" || return 1
sleep 2

# Phase 3: Wait for heartbeat to become stale
Expand Down Expand Up @@ -1521,7 +1532,7 @@ test_journal_recovery_idempotency() {

# Phase 2: Kill Node A
log "Phase 2: Killing Node A..."
kill_node "$node_a"
kill_node "$node_a" || return 1
sleep 2

# Phase 3: Wait for heartbeat timeout
Expand All @@ -1547,7 +1558,7 @@ test_journal_recovery_idempotency() {

# Phase 5: Stop Node B
log "Phase 5: Stopping Node B..."
kill_node "$node_b"
kill_node "$node_b" || return 1
sleep 2

# Phase 6: Verify idempotency on Node C
Expand Down Expand Up @@ -1617,7 +1628,7 @@ test_journal_crash_recovery_file() {
sleep 1

log "Phase 2: Simulating crash by killing container..."
kill_node "$node_a"
kill_node "$node_a" || return 1
sleep 2

# Phase 3: Restart (remount) - this triggers journal recovery
Expand Down Expand Up @@ -1684,7 +1695,7 @@ test_journal_crash_recovery_directory() {
sleep 1

log "Phase 2: Simulating crash by killing container..."
kill_node "$node_a"
kill_node "$node_a" || return 1
sleep 2

# Phase 3: Restart (remount) - this triggers journal recovery
Expand Down Expand Up @@ -1764,7 +1775,7 @@ test_journal_crash_recovery_multiple_ops() {
log "All operations complete, simulating crash..."

# Kill the container to simulate crash
docker kill "$node_a" >/dev/null 2>&1 || true
kill_node "$node_a" || return 1
sleep 2

# Restart container
Expand Down Expand Up @@ -1855,7 +1866,7 @@ test_journal_crash_during_write() {
log "Wrote $write_count rapid files, crashing now..."

# Kill container while writes may be in progress
docker kill "$node_a" >/dev/null 2>&1 || true
kill_node "$node_a" || return 1

# Wait for any background processes to terminate
wait 2>/dev/null || true
Expand Down Expand Up @@ -1973,7 +1984,7 @@ test_container_kill_recovery() {
write_file "$victim" "before_kill.txt" "Written before kill"

# Kill the victim
kill_node "$victim"
kill_node "$victim" || return 1
sleep 2

# Survivor should still be able to read the file
Expand Down Expand Up @@ -2530,7 +2541,7 @@ test_stress_random_failures() {
if [ $((RANDOM % 5)) -eq 0 ]; then
local kill_idx=$((RANDOM % NODE_COUNT))
local kill_node="${NODE_ARRAY[$kill_idx]}"
kill_node "$kill_node"
kill_node "$kill_node" || return 1
sleep 1
restart_node "$kill_node"
wait_node_healthy "$kill_node" 30
Expand Down
17 changes: 15 additions & 2 deletions tests/integration/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -522,12 +522,25 @@ func (m *MountedFS) PID() int {
return 0
}

// Kill sends SIGKILL to the mount process to simulate a crash.
// Kill sends SIGKILL to the mount process to simulate a crash and waits until it is gone.
func (m *MountedFS) Kill() error {
if m.MountCmd == nil || m.MountCmd.Process == nil {
return fmt.Errorf("mount process not running")
}
return m.MountCmd.Process.Kill()
if err := m.MountCmd.Process.Kill(); err != nil {
return err
}
exited := make(chan struct{})
go func() {
_ = m.MountCmd.Wait()
close(exited)
}()
select {
case <-exited:
return nil
case <-time.After(10 * time.Second):
return fmt.Errorf("mount process %d still running 10s after SIGKILL", m.MountCmd.Process.Pid)
}
}

// OpenFile opens a file in the mounted filesystem for read/write operations.
Expand Down
3 changes: 0 additions & 3 deletions tests/integration/multinode_helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,9 +123,6 @@ func (c *MultiNodeCluster) KillNode(idx int) error {
return err
}

// Wait briefly for process to die and unmount to propagate
time.Sleep(500 * time.Millisecond)

// Force unmount to clean up mount point
c.Nodes[idx].Unmount()

Expand Down
33 changes: 31 additions & 2 deletions tests/vm/scripts/run-vm-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -609,7 +609,11 @@ test_restart_keeps_peer_journal() {

local slot_before
slot_before=$(ssh_run "$vm_a" "grep -o 'Registered as node [0-9]*' /var/log/goclusterfs.log | tail -1")
ssh_run "$vm_a" "pkill -9 goclusterfs-mount; sleep 1; umount -l /mnt/goclusterfs 2>/dev/null; true"
if ! crash_goclusterfs "$vm_a"; then
ssh_run "$vm_b" "touch /tmp/peer-writer.stop"
return 1
fi
ssh_run "$vm_a" "umount -l /mnt/goclusterfs 2>/dev/null; true"
if ! setup_goclusterfs "$vm_a" false; then
ssh_run "$vm_b" "touch /tmp/peer-writer.stop"
return 1
Expand Down Expand Up @@ -2946,13 +2950,38 @@ test_concurrent_writes_power_failure() {
return 0
}

# Print the PIDs of goclusterfs-mount on VM $1: processes with an argument equal to the path setup_goclusterfs starts
# it with. On Alpine the gcompat loader runs it, so its name and argv[0] are the loader's. A zombie has no arguments.
mount_pids() {
ssh_run "$1" 'for p in /proc/[0-9]*; do tr "\0" "\n" 2>/dev/null <$p/cmdline | grep -qx "/usr/local/bin/goclusterfs-moun[t]" && echo ${p#/proc/}; done; true'
}

# Kill goclusterfs-mount on VM $1 with SIGKILL, as a crash would, and fail unless it is gone within 10s.
crash_goclusterfs() {
local vm_id=$1 pids waited=0
pids=$(mount_pids "$vm_id")
if [[ -z "$pids" ]]; then
log_error "VM $vm_id: no goclusterfs-mount process to kill"
return 1
fi
ssh_run "$vm_id" "kill -9 $(echo $pids)"
while [[ -n "$(mount_pids "$vm_id")" ]]; do
if [[ $waited -ge 50 ]]; then
log_error "VM $vm_id: goclusterfs-mount still running 10s after SIGKILL"
return 1
fi
sleep 0.2
waited=$((waited + 1))
done
}

# Unmount GoClusterFS on VM $1 and wait for the mount process to exit, which
# marks its journal clean.
unmount_goclusterfs() {
local vm_id=$1
ssh_run "$vm_id" "fusermount3 -u ${GOCLUSTERFS_MOUNT}" || return 1
local waited=0
while ssh_run "$vm_id" "pidof goclusterfs-mount > /dev/null"; do
while [[ -n "$(mount_pids "$vm_id")" ]]; do
if [[ $waited -ge 30 ]]; then
return 1
fi
Expand Down
9 changes: 9 additions & 0 deletions tests/vm/vm-test-harness.sh
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,15 @@ kill_vm() {

if is_vm_running "$vm_id"; then
kill -9 "$(cat "$pid_file")" 2>/dev/null || true
local waited=0
while is_vm_running "$vm_id"; do
if [[ $waited -ge 50 ]]; then
log_error "VM $vm_id: QEMU still running 10s after SIGKILL"
return 1
fi
sleep 0.2
waited=$((waited + 1))
done
mark_vm_log "$vm_id" "VM $vm_id killed"
fi
rm -f "$pid_file"
Expand Down
Loading