diff --git a/internal/node/registry.go b/internal/node/registry.go index 025c7544..8b963119 100644 --- a/internal/node/registry.go +++ b/internal/node/registry.go @@ -436,18 +436,29 @@ func (r *Registry) StartHeartbeat() error { return nil } +// testHeartbeatHold, from GOCLUSTERFS_TEST_HEARTBEAT_HOLD and for tests only, +// is how long the heartbeat waits once it finds it missed beats, as after a +// pause, so a write that was stalled with it reaches the write gate first. +var testHeartbeatHold, _ = time.ParseDuration(os.Getenv("GOCLUSTERFS_TEST_HEARTBEAT_HOLD")) + // heartbeatLoop periodically updates the node's heartbeat. func (r *Registry) heartbeatLoop(slotIdx int) { defer r.heartbeatWg.Done() ticker := time.NewTicker(HeartbeatInterval) defer ticker.Stop() + if testHeartbeatHold > 0 { + log.Printf("[WARN] node: GOCLUSTERFS_TEST_HEARTBEAT_HOLD is set: after a pause the heartbeat waits %v; use it only in tests", testHeartbeatHold) + } for { select { case <-r.stopCh: return case <-ticker.C: + if testHeartbeatHold > 0 && r.sinceLastBeat() > HeartbeatTimeout/2 { + time.Sleep(testHeartbeatHold) + } _ = r.beat(slotIdx) } } diff --git a/tests/integration/helpers.go b/tests/integration/helpers.go index 6858e9f1..b92cf443 100644 --- a/tests/integration/helpers.go +++ b/tests/integration/helpers.go @@ -24,6 +24,7 @@ type MountedFS struct { MountPoint string ExtraArgs []string // Appended to the mount command by Mount and FormatAndMount Log io.Writer // Also receives the mount process output when set + Env []string // Added to the mount process environment MountCmd *exec.Cmd BinaryPath string t *testing.T @@ -256,6 +257,7 @@ func (m *MountedFS) Mount(debug bool) error { m.MountCmd = exec.CommandContext(m.ctx, m.BinaryPath, args...) m.MountCmd.Stdout = m.output() m.MountCmd.Stderr = m.MountCmd.Stdout + m.MountCmd.Env = m.environ() if err := m.MountCmd.Start(); err != nil { return fmt.Errorf("failed to start mount command: %v", err) @@ -288,6 +290,7 @@ func (m *MountedFS) FormatAndMount(debug bool) error { m.MountCmd = exec.CommandContext(m.ctx, m.BinaryPath, args...) m.MountCmd.Stdout = m.output() m.MountCmd.Stderr = m.MountCmd.Stdout + m.MountCmd.Env = m.environ() if err := m.MountCmd.Start(); err != nil { return fmt.Errorf("failed to start mount command: %v", err) @@ -395,6 +398,14 @@ func (m *MountedFS) IsMounted() bool { return strings.Contains(string(data), m.MountPoint) } +// environ is the mount process environment: the test's, plus Env. +func (m *MountedFS) environ() []string { + if len(m.Env) == 0 { + return nil + } + return append(os.Environ(), m.Env...) +} + // Path returns the full path for a file relative to the mount point. func (m *MountedFS) Path(name string) string { return filepath.Join(m.MountPoint, name) @@ -565,6 +576,7 @@ func (m *MountedFS) FormatAndMountWithSFB(debug bool) error { m.MountCmd = exec.CommandContext(m.ctx, m.BinaryPath, args...) m.MountCmd.Stdout = m.output() m.MountCmd.Stderr = m.MountCmd.Stdout + m.MountCmd.Env = m.environ() if err := m.MountCmd.Start(); err != nil { return fmt.Errorf("failed to start mount command: %v", err) @@ -598,6 +610,7 @@ func (m *MountedFS) MountWithSFB(debug bool) error { m.MountCmd = exec.CommandContext(m.ctx, m.BinaryPath, args...) m.MountCmd.Stdout = m.output() m.MountCmd.Stderr = m.MountCmd.Stdout + m.MountCmd.Env = m.environ() if err := m.MountCmd.Start(); err != nil { return fmt.Errorf("failed to start mount command: %v", err) diff --git a/tests/integration/lock_reclaim_test.go b/tests/integration/lock_reclaim_test.go index 9250b9ac..724b7967 100644 --- a/tests/integration/lock_reclaim_test.go +++ b/tests/integration/lock_reclaim_test.go @@ -1,10 +1,12 @@ package integration import ( + "bytes" "context" "fmt" "regexp" "strconv" + "strings" "syscall" "testing" "time" @@ -147,14 +149,109 @@ func TestWaiterOutlastsLockTimeoutThroughFailover(t *testing.T) { } } +// TestStalledNodesWritesStopAtWriteGate stops node A while a writer keeps +// writing a file through it, waits until node B has proven A dead and replayed +// it, then writes the file from B and resumes A. A's heartbeat is held back after +// the pause, so A's queued write reaches the device first, where the write gate +// must refuse it: A exits, and B's data stays. +func TestStalledNodesWritesStopAtWriteGate(t *testing.T) { + cluster, nodeA, _ := startPairForLockTestWithEnv(t, []string{"GOCLUSTERFS_TEST_HEARTBEAT_HOLD=30s"}) + a, b := cluster.Node(0), cluster.Node(1) + logA := a.Log.(*syncBuffer) + pid := a.PID() + if err := b.WriteFile("fenced", bytes.Repeat([]byte{'0'}, 4096)); err != nil { + t.Fatal(err) + } + + stop := make(chan struct{}) + defer close(stop) + go func() { + for { + select { + case <-stop: + return + default: + } + if a.WriteFileAt("fenced", bytes.Repeat([]byte{'A'}, 4096), 0) != nil { + return + } + } + }() + time.Sleep(time.Second) + if err := syscall.Kill(pid, syscall.SIGSTOP); err != nil { + t.Fatalf("stop node A: %v", err) + } + defer cluster.KillNode(0) + + waitSlotEmpty(t, cluster.ImagePath, nodeA.slot, 90*time.Second) + if err := b.WriteFileAt("fenced", bytes.Repeat([]byte{'B'}, 4096), 0); err != nil { + t.Fatalf("node B writes after taking over: %v", err) + } + if err := syscall.Kill(pid, syscall.SIGCONT); err != nil { + t.Fatalf("resume node A: %v", err) + } + + // The held heartbeat stops A after 30s whatever the gate does; by then a write the gate let through has landed. + lost := regexp.MustCompile(fmt.Sprintf(`Node registry slot %d is no longer this node's`, nodeA.slot)) + deadline := time.Now().Add(45 * time.Second) + for !lost.MatchString(logA.String()) && time.Now().Before(deadline) { + time.Sleep(100 * time.Millisecond) + } + got, err := b.ReadFile("fenced") + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, bytes.Repeat([]byte{'B'}, 4096)) { + t.Fatalf("file reads %q... after node A resumed, want node B's data", got[:16]) + } + if !lost.MatchString(logA.String()) { + t.Fatal("node A did not stop after resuming from its takeover") + } + if !strings.Contains(logA.String(), "checking the slot before a write") { + t.Error("node A stopped without its write gate checking the slot") + } +} + +// waitSlotEmpty waits until registry slot slot is empty: a peer replayed its node. +func waitSlotEmpty(t *testing.T, imagePath string, slot int, timeout time.Duration) { + t.Helper() + dev, err := storage.Open(imagePath) + if err != nil { + t.Fatal(err) + } + defer dev.Close() + sb, err := superblock.Read(dev) + if err != nil { + t.Fatal(err) + } + reg := node.NewRegistry(dev, sb.NodeRegistryOffset) + deadline := time.Now().Add(timeout) + for { + if _, state := reg.Occupant(slot, true); state == node.Unoccupied { + return + } + if time.Now().After(deadline) { + t.Fatalf("registry slot %d still occupied after %v", slot, timeout) + } + time.Sleep(500 * time.Millisecond) + } +} + type registeredNode struct { uuid [16]byte slot int } func startPairForLockTest(t *testing.T) (*MultiNodeCluster, registeredNode, *syncBuffer) { + t.Helper() + return startPairForLockTestWithEnv(t, nil) +} + +// startPairForLockTestWithEnv is startPairForLockTest with envA added to node A's environment. +func startPairForLockTestWithEnv(t *testing.T, envA []string) (*MultiNodeCluster, registeredNode, *syncBuffer) { t.Helper() cluster := NewMultiNodeCluster(t, 2, 50) + cluster.Node(0).Env = envA logA, logB := &syncBuffer{}, &syncBuffer{} cluster.Node(0).Log, cluster.Node(1).Log = logA, logB if err := cluster.FormatAndStartFirst(false); err != nil { diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index d3eb4337..1786c678 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -637,6 +637,15 @@ scenarios: docker: TODO # needs a lock record written as the stopped node at a known moment; Go-only harness vm: TODO # same as docker + - id: stalled-node-writes-stop-at-write-gate + name: Stalled node's write stops at the write gate + description: Node A is stopped while a writer writes through it; after node B replays A and writes the file, A resumes with its heartbeat held, so its queued write reaches the write gate first, which refuses it and stops A, and the file keeps B's data byte for byte + priority: high + environments: + go: lock_reclaim_test.go::TestStalledNodesWritesStopAtWriteGate + docker: TODO # needs SIGSTOP and SIGCONT on one container's mount process with a writer through it; covered in Go and VM + vm: run-vm-tests.sh::test_stalled_node_writes_stop_at_write_gate + - id: lock-adopt-owner name: Replaying node takes over a dead node's exclusive locks description: AdoptOwner holds the dead node's exclusive records through the replay and ReleaseAdopted frees them; shared records and other nodes' records stay diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 3fce60e5..3f1a2d64 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -50,6 +50,10 @@ test_failed=0 run_test() { local test_name="$1" local test_func="$2" + # VM_TEST_ONLY runs only the tests whose names contain it + if [[ -n "${VM_TEST_ONLY:-}" && "$test_name" != *"$VM_TEST_ONLY"* ]]; then + return 0 + fi log_info "==========================================" log_info "Running test: $test_name" @@ -2987,6 +2991,86 @@ unmount_goclusterfs() { done } +# A node stopped while a writer writes through it is proven dead and replayed by +# a peer, and the peer writes the file. When the stopped node resumes, its +# heartbeat is held (GOCLUSTERFS_TEST_HEARTBEAT_HOLD), so its queued write reaches +# the write gate first. The gate must refuse it and stop the node, and the +# file must keep the peer's data byte for byte. +test_stalled_node_writes_stop_at_write_gate() { + log_info "Testing that a stalled node's queued write stops at the write gate..." + if [[ $NODE_COUNT -lt 2 ]]; then + log_warn "Skipping: Need at least 2 VMs for this test" + return 0 + fi + ensure_cluster_healthy "$NODE_COUNT" + local stalled=1 peer=2 pids waited + unmount_goclusterfs "$stalled" || { log_error "VM $stalled: unmount failed"; return 1; } + setup_goclusterfs "$stalled" false "GOCLUSTERFS_TEST_HEARTBEAT_HOLD=30s" + + ssh_run "$peer" "head -c 4096 /dev/zero | tr '\\000' 0 > ${GOCLUSTERFS_MOUNT}/fenced && sync" + ssh_run "$stalled" "head -c 4096 /dev/zero | tr '\\000' A > /tmp/fenced.A" + ssh_run "$peer" "head -c 4096 /dev/zero | tr '\\000' B > /tmp/fenced.B" + ssh_run "$stalled" "nohup sh -c 'while dd if=/tmp/fenced.A of=${GOCLUSTERFS_MOUNT}/fenced bs=4096 count=1 conv=notrunc 2>/dev/null; do :; done' < /dev/null > /dev/null 2>&1 &" + sleep 1 + + local marker="stalled-node gate test: stopping VM $stalled $(date +%s)" + mark_vm_logs "$marker" + pids=$(mount_pids "$stalled") + ssh_run "$stalled" "kill -STOP $(echo $pids)" + local peer_log stalled_log + peer_log=$(get_vm_mount_log "$peer") + stalled_log=$(get_vm_mount_log "$stalled") + waited=0 + until sed -n "/$marker/,\$p" "$peer_log" | grep -q "Starting journal replay for node"; do + if [[ $waited -ge 120 ]]; then + log_error "VM $peer did not start replaying VM $stalled within 120s" + ssh_run "$stalled" "kill -CONT $(echo $pids)" + return 1 + fi + sleep 1 + waited=$((waited + 1)) + done + sleep 5 + if ! ssh_run "$peer" "dd if=/tmp/fenced.B of=${GOCLUSTERFS_MOUNT}/fenced bs=4096 count=1 conv=notrunc 2>/dev/null && sync"; then + log_error "VM $peer could not write after taking over" + ssh_run "$stalled" "kill -CONT $(echo $pids)" + return 1 + fi + ssh_run "$stalled" "kill -CONT $(echo $pids)" + + # The held heartbeat stops the node after 30s whatever the gate does; by then a write the gate let through has landed. + waited=0 + while [[ -n "$(mount_pids "$stalled")" && $waited -lt 45 ]]; do + sleep 1 + waited=$((waited + 1)) + done + local result=0 + if ! ssh_run "$peer" "cmp -s ${GOCLUSTERFS_MOUNT}/fenced /tmp/fenced.B"; then + log_error "fenced does not hold VM $peer's data after VM $stalled resumed" + result=1 + fi + if [[ -n "$(mount_pids "$stalled")" ]]; then + log_error "VM $stalled did not stop after resuming from its takeover" + result=1 + fi + if ! sed -n "/$marker/,\$p" "$stalled_log" | grep -q "checking the slot before a write"; then + log_error "VM $stalled stopped without its write gate checking the slot" + result=1 + fi + + if [[ -n "$(mount_pids "$stalled")" ]]; then + crash_goclusterfs "$stalled" || true + fi + # The bracket keeps pkill from matching the shell that runs it. + ssh_run "$stalled" "pkill -f 'of=${GOCLUSTERFS_MOUNT}/fence[d]'; fusermount3 -uz ${GOCLUSTERFS_MOUNT}" || true + if ! setup_goclusterfs "$stalled" false; then + log_error "VM $stalled: remount after the test failed" + result=1 + fi + [[ $result -eq 0 ]] && log_info "Stalled node's queued write stopped at the write gate" + return $result +} + # A power failure while files are written may leave blocks that no file maps. # With every node unmounted, fsck -repair frees any it finds, the check after # it is clean, and no file changes. @@ -3538,6 +3622,7 @@ main() { run_test "Reversed Write Order" test_reversed_write_order run_test "Concurrent Writes Power Failure" test_concurrent_writes_power_failure run_test "fsck Repair After Power Failure" test_fsck_repair_after_power_failure + run_test "Stalled Node Writes Stop At Write Gate" test_stalled_node_writes_stop_at_write_gate # Summary log_info "==========================================" diff --git a/tests/vm/vm-test-harness.sh b/tests/vm/vm-test-harness.sh index b3a881ca..09d7a837 100755 --- a/tests/vm/vm-test-harness.sh +++ b/tests/vm/vm-test-harness.sh @@ -512,6 +512,8 @@ report_boot_failure() { setup_goclusterfs() { local vm_id="$1" local format_disk="${2:-false}" + # Extra environment for the mount process, such as a test failpoint + local mount_env="${3:-}" local ssh_port=$(get_vm_ssh_port "$vm_id") log_info "Setting up GoClusterFS on VM $vm_id (format=$format_disk)..." @@ -535,7 +537,7 @@ setup_goclusterfs() { # /var/log/goclusterfs.log is lost when the VM dies; the log device keeps a copy on the host log_info "Mounting GoClusterFS..." ssh_run "$vm_id" \ - "nohup /usr/local/bin/goclusterfs-mount $mount_args < /dev/null 2>&1 | tee /var/log/goclusterfs.log > /dev/goclusterfs-log 2>/dev/null &" + "nohup env $mount_env /usr/local/bin/goclusterfs-mount $mount_args < /dev/null 2>&1 | tee /var/log/goclusterfs.log > /dev/goclusterfs-log 2>/dev/null &" # Wait for mount to be ready wait_for_goclusterfs_mount "$vm_id"