From 981ab99aa618a3b04aecfb7faf07499f2ad360ec Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 14:35:31 -0600 Subject: [PATCH 1/2] test: prove the write gate stops a stalled node's queued write A node stopped while a writer writes through it resumes after a peer has replayed it and written the file. GOCLUSTERFS_TEST_HEARTBEAT_HOLD, a test-only setting, holds its heartbeat after the pause, so the queued write reaches the device before the heartbeat notices the lost slot. The Go and VM tests check that the write gate stops the node and the peer's data stays byte for byte. Without the gate both fail: the stalled write lands. Co-Authored-By: Claude Opus 5.5 --- internal/node/registry.go | 11 +++ tests/integration/helpers.go | 13 ++++ tests/integration/lock_reclaim_test.go | 97 ++++++++++++++++++++++++++ tests/test-manifest.yaml | 9 +++ tests/vm/scripts/run-vm-tests.sh | 81 +++++++++++++++++++++ tests/vm/vm-test-harness.sh | 4 +- 6 files changed, 214 insertions(+), 1 deletion(-) diff --git a/internal/node/registry.go b/internal/node/registry.go index 025c7544..8b963119 100644 --- a/internal/node/registry.go +++ b/internal/node/registry.go @@ -436,18 +436,29 @@ func (r *Registry) StartHeartbeat() error { return nil } +// testHeartbeatHold, from GOCLUSTERFS_TEST_HEARTBEAT_HOLD and for tests only, +// is how long the heartbeat waits once it finds it missed beats, as after a +// pause, so a write that was stalled with it reaches the write gate first. +var testHeartbeatHold, _ = time.ParseDuration(os.Getenv("GOCLUSTERFS_TEST_HEARTBEAT_HOLD")) + // heartbeatLoop periodically updates the node's heartbeat. func (r *Registry) heartbeatLoop(slotIdx int) { defer r.heartbeatWg.Done() ticker := time.NewTicker(HeartbeatInterval) defer ticker.Stop() + if testHeartbeatHold > 0 { + log.Printf("[WARN] node: GOCLUSTERFS_TEST_HEARTBEAT_HOLD is set: after a pause the heartbeat waits %v; use it only in tests", testHeartbeatHold) + } for { select { case <-r.stopCh: return case <-ticker.C: + if testHeartbeatHold > 0 && r.sinceLastBeat() > HeartbeatTimeout/2 { + time.Sleep(testHeartbeatHold) + } _ = r.beat(slotIdx) } } diff --git a/tests/integration/helpers.go b/tests/integration/helpers.go index 6858e9f1..b92cf443 100644 --- a/tests/integration/helpers.go +++ b/tests/integration/helpers.go @@ -24,6 +24,7 @@ type MountedFS struct { MountPoint string ExtraArgs []string // Appended to the mount command by Mount and FormatAndMount Log io.Writer // Also receives the mount process output when set + Env []string // Added to the mount process environment MountCmd *exec.Cmd BinaryPath string t *testing.T @@ -256,6 +257,7 @@ func (m *MountedFS) Mount(debug bool) error { m.MountCmd = exec.CommandContext(m.ctx, m.BinaryPath, args...) m.MountCmd.Stdout = m.output() m.MountCmd.Stderr = m.MountCmd.Stdout + m.MountCmd.Env = m.environ() if err := m.MountCmd.Start(); err != nil { return fmt.Errorf("failed to start mount command: %v", err) @@ -288,6 +290,7 @@ func (m *MountedFS) FormatAndMount(debug bool) error { m.MountCmd = exec.CommandContext(m.ctx, m.BinaryPath, args...) m.MountCmd.Stdout = m.output() m.MountCmd.Stderr = m.MountCmd.Stdout + m.MountCmd.Env = m.environ() if err := m.MountCmd.Start(); err != nil { return fmt.Errorf("failed to start mount command: %v", err) @@ -395,6 +398,14 @@ func (m *MountedFS) IsMounted() bool { return strings.Contains(string(data), m.MountPoint) } +// environ is the mount process environment: the test's, plus Env. +func (m *MountedFS) environ() []string { + if len(m.Env) == 0 { + return nil + } + return append(os.Environ(), m.Env...) +} + // Path returns the full path for a file relative to the mount point. func (m *MountedFS) Path(name string) string { return filepath.Join(m.MountPoint, name) @@ -565,6 +576,7 @@ func (m *MountedFS) FormatAndMountWithSFB(debug bool) error { m.MountCmd = exec.CommandContext(m.ctx, m.BinaryPath, args...) m.MountCmd.Stdout = m.output() m.MountCmd.Stderr = m.MountCmd.Stdout + m.MountCmd.Env = m.environ() if err := m.MountCmd.Start(); err != nil { return fmt.Errorf("failed to start mount command: %v", err) @@ -598,6 +610,7 @@ func (m *MountedFS) MountWithSFB(debug bool) error { m.MountCmd = exec.CommandContext(m.ctx, m.BinaryPath, args...) m.MountCmd.Stdout = m.output() m.MountCmd.Stderr = m.MountCmd.Stdout + m.MountCmd.Env = m.environ() if err := m.MountCmd.Start(); err != nil { return fmt.Errorf("failed to start mount command: %v", err) diff --git a/tests/integration/lock_reclaim_test.go b/tests/integration/lock_reclaim_test.go index 9250b9ac..724b7967 100644 --- a/tests/integration/lock_reclaim_test.go +++ b/tests/integration/lock_reclaim_test.go @@ -1,10 +1,12 @@ package integration import ( + "bytes" "context" "fmt" "regexp" "strconv" + "strings" "syscall" "testing" "time" @@ -147,14 +149,109 @@ func TestWaiterOutlastsLockTimeoutThroughFailover(t *testing.T) { } } +// TestStalledNodesWritesStopAtWriteGate stops node A while a writer keeps +// writing a file through it, waits until node B has proven A dead and replayed +// it, then writes the file from B and resumes A. A's heartbeat is held back after +// the pause, so A's queued write reaches the device first, where the write gate +// must refuse it: A exits, and B's data stays. +func TestStalledNodesWritesStopAtWriteGate(t *testing.T) { + cluster, nodeA, _ := startPairForLockTestWithEnv(t, []string{"GOCLUSTERFS_TEST_HEARTBEAT_HOLD=30s"}) + a, b := cluster.Node(0), cluster.Node(1) + logA := a.Log.(*syncBuffer) + pid := a.PID() + if err := b.WriteFile("fenced", bytes.Repeat([]byte{'0'}, 4096)); err != nil { + t.Fatal(err) + } + + stop := make(chan struct{}) + defer close(stop) + go func() { + for { + select { + case <-stop: + return + default: + } + if a.WriteFileAt("fenced", bytes.Repeat([]byte{'A'}, 4096), 0) != nil { + return + } + } + }() + time.Sleep(time.Second) + if err := syscall.Kill(pid, syscall.SIGSTOP); err != nil { + t.Fatalf("stop node A: %v", err) + } + defer cluster.KillNode(0) + + waitSlotEmpty(t, cluster.ImagePath, nodeA.slot, 90*time.Second) + if err := b.WriteFileAt("fenced", bytes.Repeat([]byte{'B'}, 4096), 0); err != nil { + t.Fatalf("node B writes after taking over: %v", err) + } + if err := syscall.Kill(pid, syscall.SIGCONT); err != nil { + t.Fatalf("resume node A: %v", err) + } + + // The held heartbeat stops A after 30s whatever the gate does; by then a write the gate let through has landed. + lost := regexp.MustCompile(fmt.Sprintf(`Node registry slot %d is no longer this node's`, nodeA.slot)) + deadline := time.Now().Add(45 * time.Second) + for !lost.MatchString(logA.String()) && time.Now().Before(deadline) { + time.Sleep(100 * time.Millisecond) + } + got, err := b.ReadFile("fenced") + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, bytes.Repeat([]byte{'B'}, 4096)) { + t.Fatalf("file reads %q... after node A resumed, want node B's data", got[:16]) + } + if !lost.MatchString(logA.String()) { + t.Fatal("node A did not stop after resuming from its takeover") + } + if !strings.Contains(logA.String(), "checking the slot before a write") { + t.Error("node A stopped without its write gate checking the slot") + } +} + +// waitSlotEmpty waits until registry slot slot is empty: a peer replayed its node. +func waitSlotEmpty(t *testing.T, imagePath string, slot int, timeout time.Duration) { + t.Helper() + dev, err := storage.Open(imagePath) + if err != nil { + t.Fatal(err) + } + defer dev.Close() + sb, err := superblock.Read(dev) + if err != nil { + t.Fatal(err) + } + reg := node.NewRegistry(dev, sb.NodeRegistryOffset) + deadline := time.Now().Add(timeout) + for { + if _, state := reg.Occupant(slot, true); state == node.Unoccupied { + return + } + if time.Now().After(deadline) { + t.Fatalf("registry slot %d still occupied after %v", slot, timeout) + } + time.Sleep(500 * time.Millisecond) + } +} + type registeredNode struct { uuid [16]byte slot int } func startPairForLockTest(t *testing.T) (*MultiNodeCluster, registeredNode, *syncBuffer) { + t.Helper() + return startPairForLockTestWithEnv(t, nil) +} + +// startPairForLockTestWithEnv is startPairForLockTest with envA added to node A's environment. +func startPairForLockTestWithEnv(t *testing.T, envA []string) (*MultiNodeCluster, registeredNode, *syncBuffer) { t.Helper() cluster := NewMultiNodeCluster(t, 2, 50) + cluster.Node(0).Env = envA logA, logB := &syncBuffer{}, &syncBuffer{} cluster.Node(0).Log, cluster.Node(1).Log = logA, logB if err := cluster.FormatAndStartFirst(false); err != nil { diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index d3eb4337..1786c678 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -637,6 +637,15 @@ scenarios: docker: TODO # needs a lock record written as the stopped node at a known moment; Go-only harness vm: TODO # same as docker + - id: stalled-node-writes-stop-at-write-gate + name: Stalled node's write stops at the write gate + description: Node A is stopped while a writer writes through it; after node B replays A and writes the file, A resumes with its heartbeat held, so its queued write reaches the write gate first, which refuses it and stops A, and the file keeps B's data byte for byte + priority: high + environments: + go: lock_reclaim_test.go::TestStalledNodesWritesStopAtWriteGate + docker: TODO # needs SIGSTOP and SIGCONT on one container's mount process with a writer through it; covered in Go and VM + vm: run-vm-tests.sh::test_stalled_node_writes_stop_at_write_gate + - id: lock-adopt-owner name: Replaying node takes over a dead node's exclusive locks description: AdoptOwner holds the dead node's exclusive records through the replay and ReleaseAdopted frees them; shared records and other nodes' records stay diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 3fce60e5..e8ccd7a3 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -50,6 +50,10 @@ test_failed=0 run_test() { local test_name="$1" local test_func="$2" + # VM_TEST_ONLY runs only the tests whose names contain it + if [[ -n "${VM_TEST_ONLY:-}" && "$test_name" != *"$VM_TEST_ONLY"* ]]; then + return 0 + fi log_info "==========================================" log_info "Running test: $test_name" @@ -2987,6 +2991,82 @@ unmount_goclusterfs() { done } +# A node stopped while a writer writes through it is proven dead and replayed by +# a peer, and the peer writes the file. When the stopped node resumes, its +# heartbeat is held (GOCLUSTERFS_TEST_HEARTBEAT_HOLD), so its queued write reaches +# the write gate first. The gate must refuse it and stop the node, and the +# file must keep the peer's data byte for byte. +test_stalled_node_writes_stop_at_write_gate() { + log_info "Testing that a stalled node's queued write stops at the write gate..." + if [[ $NODE_COUNT -lt 2 ]]; then + log_warn "Skipping: Need at least 2 VMs for this test" + return 0 + fi + ensure_cluster_healthy "$NODE_COUNT" + local stalled=1 peer=2 pids waited + unmount_goclusterfs "$stalled" || { log_error "VM $stalled: unmount failed"; return 1; } + setup_goclusterfs "$stalled" false "GOCLUSTERFS_TEST_HEARTBEAT_HOLD=30s" + + ssh_run "$peer" "head -c 4096 /dev/zero | tr '\\000' 0 > ${GOCLUSTERFS_MOUNT}/fenced && sync" + ssh_run "$stalled" "head -c 4096 /dev/zero | tr '\\000' A > /tmp/fenced.A" + ssh_run "$peer" "head -c 4096 /dev/zero | tr '\\000' B > /tmp/fenced.B" + ssh_run "$stalled" "nohup sh -c 'while dd if=/tmp/fenced.A of=${GOCLUSTERFS_MOUNT}/fenced bs=4096 count=1 conv=notrunc 2>/dev/null; do :; done' < /dev/null > /dev/null 2>&1 &" + sleep 1 + + local marker="stalled-node gate test: stopping VM $stalled $(date +%s)" + mark_vm_logs "$marker" + pids=$(mount_pids "$stalled") + ssh_run "$stalled" "kill -STOP $(echo $pids)" + local peer_log stalled_log + peer_log=$(get_vm_mount_log "$peer") + stalled_log=$(get_vm_mount_log "$stalled") + waited=0 + until sed -n "/$marker/,\$p" "$peer_log" | grep -q "Starting journal replay for node"; do + if [[ $waited -ge 120 ]]; then + log_error "VM $peer did not start replaying VM $stalled within 120s" + ssh_run "$stalled" "kill -CONT $(echo $pids)" + return 1 + fi + sleep 1 + waited=$((waited + 1)) + done + sleep 5 + if ! ssh_run "$peer" "dd if=/tmp/fenced.B of=${GOCLUSTERFS_MOUNT}/fenced bs=4096 count=1 conv=notrunc 2>/dev/null && sync"; then + log_error "VM $peer could not write after taking over" + ssh_run "$stalled" "kill -CONT $(echo $pids)" + return 1 + fi + ssh_run "$stalled" "kill -CONT $(echo $pids)" + + # The held heartbeat stops the node after 30s whatever the gate does; by then a write the gate let through has landed. + waited=0 + while [[ -n "$(mount_pids "$stalled")" && $waited -lt 45 ]]; do + sleep 1 + waited=$((waited + 1)) + done + local result=0 + if ! ssh_run "$peer" "cmp -s ${GOCLUSTERFS_MOUNT}/fenced /tmp/fenced.B"; then + log_error "fenced does not hold VM $peer's data after VM $stalled resumed" + result=1 + fi + if [[ -n "$(mount_pids "$stalled")" ]]; then + log_error "VM $stalled did not stop after resuming from its takeover" + result=1 + fi + if ! sed -n "/$marker/,\$p" "$stalled_log" | grep -q "checking the slot before a write"; then + log_error "VM $stalled stopped without its write gate checking the slot" + result=1 + fi + + if [[ -n "$(mount_pids "$stalled")" ]]; then + crash_goclusterfs "$stalled" || true + fi + ssh_run "$stalled" "pkill -f 'of=${GOCLUSTERFS_MOUNT}/fenced'; fusermount3 -uz ${GOCLUSTERFS_MOUNT}" || true + setup_goclusterfs "$stalled" false + [[ $result -eq 0 ]] && log_info "Stalled node's queued write stopped at the write gate" + return $result +} + # A power failure while files are written may leave blocks that no file maps. # With every node unmounted, fsck -repair frees any it finds, the check after # it is clean, and no file changes. @@ -3538,6 +3618,7 @@ main() { run_test "Reversed Write Order" test_reversed_write_order run_test "Concurrent Writes Power Failure" test_concurrent_writes_power_failure run_test "fsck Repair After Power Failure" test_fsck_repair_after_power_failure + run_test "Stalled Node Writes Stop At Write Gate" test_stalled_node_writes_stop_at_write_gate # Summary log_info "==========================================" diff --git a/tests/vm/vm-test-harness.sh b/tests/vm/vm-test-harness.sh index b3a881ca..09d7a837 100755 --- a/tests/vm/vm-test-harness.sh +++ b/tests/vm/vm-test-harness.sh @@ -512,6 +512,8 @@ report_boot_failure() { setup_goclusterfs() { local vm_id="$1" local format_disk="${2:-false}" + # Extra environment for the mount process, such as a test failpoint + local mount_env="${3:-}" local ssh_port=$(get_vm_ssh_port "$vm_id") log_info "Setting up GoClusterFS on VM $vm_id (format=$format_disk)..." @@ -535,7 +537,7 @@ setup_goclusterfs() { # /var/log/goclusterfs.log is lost when the VM dies; the log device keeps a copy on the host log_info "Mounting GoClusterFS..." ssh_run "$vm_id" \ - "nohup /usr/local/bin/goclusterfs-mount $mount_args < /dev/null 2>&1 | tee /var/log/goclusterfs.log > /dev/goclusterfs-log 2>/dev/null &" + "nohup env $mount_env /usr/local/bin/goclusterfs-mount $mount_args < /dev/null 2>&1 | tee /var/log/goclusterfs.log > /dev/goclusterfs-log 2>/dev/null &" # Wait for mount to be ready wait_for_goclusterfs_mount "$vm_id" From a814a632a08d4bae36eab992fff76ae713cadfe5 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 15:12:38 -0600 Subject: [PATCH 2/2] test(vm): clear the stalled node's mount before remounting it Co-Authored-By: Claude Opus 5.5 --- tests/vm/scripts/run-vm-tests.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index e8ccd7a3..3f1a2d64 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -3061,8 +3061,12 @@ test_stalled_node_writes_stop_at_write_gate() { if [[ -n "$(mount_pids "$stalled")" ]]; then crash_goclusterfs "$stalled" || true fi - ssh_run "$stalled" "pkill -f 'of=${GOCLUSTERFS_MOUNT}/fenced'; fusermount3 -uz ${GOCLUSTERFS_MOUNT}" || true - setup_goclusterfs "$stalled" false + # The bracket keeps pkill from matching the shell that runs it. + ssh_run "$stalled" "pkill -f 'of=${GOCLUSTERFS_MOUNT}/fence[d]'; fusermount3 -uz ${GOCLUSTERFS_MOUNT}" || true + if ! setup_goclusterfs "$stalled" false; then + log_error "VM $stalled: remount after the test failed" + result=1 + fi [[ $result -eq 0 ]] && log_info "Stalled node's queued write stopped at the write gate" return $result }