Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions internal/node/registry.go
Original file line number Diff line number Diff line change
Expand Up @@ -436,18 +436,29 @@ func (r *Registry) StartHeartbeat() error {
return nil
}

// testHeartbeatHold, from GOCLUSTERFS_TEST_HEARTBEAT_HOLD and for tests only,
// is how long the heartbeat waits once it finds it missed beats, as after a
// pause, so a write that was stalled with it reaches the write gate first.
var testHeartbeatHold, _ = time.ParseDuration(os.Getenv("GOCLUSTERFS_TEST_HEARTBEAT_HOLD"))

// heartbeatLoop periodically updates the node's heartbeat.
func (r *Registry) heartbeatLoop(slotIdx int) {
defer r.heartbeatWg.Done()

ticker := time.NewTicker(HeartbeatInterval)
defer ticker.Stop()
if testHeartbeatHold > 0 {
log.Printf("[WARN] node: GOCLUSTERFS_TEST_HEARTBEAT_HOLD is set: after a pause the heartbeat waits %v; use it only in tests", testHeartbeatHold)
}

for {
select {
case <-r.stopCh:
return
case <-ticker.C:
if testHeartbeatHold > 0 && r.sinceLastBeat() > HeartbeatTimeout/2 {
time.Sleep(testHeartbeatHold)
}
_ = r.beat(slotIdx)
}
}
Expand Down
13 changes: 13 additions & 0 deletions tests/integration/helpers.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ type MountedFS struct {
MountPoint string
ExtraArgs []string // Appended to the mount command by Mount and FormatAndMount
Log io.Writer // Also receives the mount process output when set
Env []string // Added to the mount process environment
MountCmd *exec.Cmd
BinaryPath string
t *testing.T
Expand Down Expand Up @@ -256,6 +257,7 @@ func (m *MountedFS) Mount(debug bool) error {
m.MountCmd = exec.CommandContext(m.ctx, m.BinaryPath, args...)
m.MountCmd.Stdout = m.output()
m.MountCmd.Stderr = m.MountCmd.Stdout
m.MountCmd.Env = m.environ()

if err := m.MountCmd.Start(); err != nil {
return fmt.Errorf("failed to start mount command: %v", err)
Expand Down Expand Up @@ -288,6 +290,7 @@ func (m *MountedFS) FormatAndMount(debug bool) error {
m.MountCmd = exec.CommandContext(m.ctx, m.BinaryPath, args...)
m.MountCmd.Stdout = m.output()
m.MountCmd.Stderr = m.MountCmd.Stdout
m.MountCmd.Env = m.environ()

if err := m.MountCmd.Start(); err != nil {
return fmt.Errorf("failed to start mount command: %v", err)
Expand Down Expand Up @@ -395,6 +398,14 @@ func (m *MountedFS) IsMounted() bool {
return strings.Contains(string(data), m.MountPoint)
}

// environ is the mount process environment: the test's, plus Env.
func (m *MountedFS) environ() []string {
if len(m.Env) == 0 {
return nil
}
return append(os.Environ(), m.Env...)
}

// Path returns the full path for a file relative to the mount point.
func (m *MountedFS) Path(name string) string {
return filepath.Join(m.MountPoint, name)
Expand Down Expand Up @@ -565,6 +576,7 @@ func (m *MountedFS) FormatAndMountWithSFB(debug bool) error {
m.MountCmd = exec.CommandContext(m.ctx, m.BinaryPath, args...)
m.MountCmd.Stdout = m.output()
m.MountCmd.Stderr = m.MountCmd.Stdout
m.MountCmd.Env = m.environ()

if err := m.MountCmd.Start(); err != nil {
return fmt.Errorf("failed to start mount command: %v", err)
Expand Down Expand Up @@ -598,6 +610,7 @@ func (m *MountedFS) MountWithSFB(debug bool) error {
m.MountCmd = exec.CommandContext(m.ctx, m.BinaryPath, args...)
m.MountCmd.Stdout = m.output()
m.MountCmd.Stderr = m.MountCmd.Stdout
m.MountCmd.Env = m.environ()

if err := m.MountCmd.Start(); err != nil {
return fmt.Errorf("failed to start mount command: %v", err)
Expand Down
97 changes: 97 additions & 0 deletions tests/integration/lock_reclaim_test.go
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package integration

import (
"bytes"
"context"
"fmt"
"regexp"
"strconv"
"strings"
"syscall"
"testing"
"time"
Expand Down Expand Up @@ -147,14 +149,109 @@ func TestWaiterOutlastsLockTimeoutThroughFailover(t *testing.T) {
}
}

// TestStalledNodesWritesStopAtWriteGate stops node A while a writer keeps
// writing a file through it, waits until node B has proven A dead and replayed
// it, then writes the file from B and resumes A. A's heartbeat is held back after
// the pause, so A's queued write reaches the device first, where the write gate
// must refuse it: A exits, and B's data stays.
func TestStalledNodesWritesStopAtWriteGate(t *testing.T) {
cluster, nodeA, _ := startPairForLockTestWithEnv(t, []string{"GOCLUSTERFS_TEST_HEARTBEAT_HOLD=30s"})
a, b := cluster.Node(0), cluster.Node(1)
logA := a.Log.(*syncBuffer)
pid := a.PID()
if err := b.WriteFile("fenced", bytes.Repeat([]byte{'0'}, 4096)); err != nil {
t.Fatal(err)
}

stop := make(chan struct{})
defer close(stop)
go func() {
for {
select {
case <-stop:
return
default:
}
if a.WriteFileAt("fenced", bytes.Repeat([]byte{'A'}, 4096), 0) != nil {
return
}
}
}()
time.Sleep(time.Second)
if err := syscall.Kill(pid, syscall.SIGSTOP); err != nil {
t.Fatalf("stop node A: %v", err)
}
defer cluster.KillNode(0)

waitSlotEmpty(t, cluster.ImagePath, nodeA.slot, 90*time.Second)
if err := b.WriteFileAt("fenced", bytes.Repeat([]byte{'B'}, 4096), 0); err != nil {
t.Fatalf("node B writes after taking over: %v", err)
}
if err := syscall.Kill(pid, syscall.SIGCONT); err != nil {
t.Fatalf("resume node A: %v", err)
}

// The held heartbeat stops A after 30s whatever the gate does; by then a write the gate let through has landed.
lost := regexp.MustCompile(fmt.Sprintf(`Node registry slot %d is no longer this node's`, nodeA.slot))
deadline := time.Now().Add(45 * time.Second)
for !lost.MatchString(logA.String()) && time.Now().Before(deadline) {
time.Sleep(100 * time.Millisecond)
}
got, err := b.ReadFile("fenced")
if err != nil {
t.Fatal(err)
}
if !bytes.Equal(got, bytes.Repeat([]byte{'B'}, 4096)) {
t.Fatalf("file reads %q... after node A resumed, want node B's data", got[:16])
}
if !lost.MatchString(logA.String()) {
t.Fatal("node A did not stop after resuming from its takeover")
}
if !strings.Contains(logA.String(), "checking the slot before a write") {
t.Error("node A stopped without its write gate checking the slot")
}
}

// waitSlotEmpty waits until registry slot slot is empty: a peer replayed its node.
func waitSlotEmpty(t *testing.T, imagePath string, slot int, timeout time.Duration) {
t.Helper()
dev, err := storage.Open(imagePath)
if err != nil {
t.Fatal(err)
}
defer dev.Close()
sb, err := superblock.Read(dev)
if err != nil {
t.Fatal(err)
}
reg := node.NewRegistry(dev, sb.NodeRegistryOffset)
deadline := time.Now().Add(timeout)
for {
if _, state := reg.Occupant(slot, true); state == node.Unoccupied {
return
}
if time.Now().After(deadline) {
t.Fatalf("registry slot %d still occupied after %v", slot, timeout)
}
time.Sleep(500 * time.Millisecond)
}
}

type registeredNode struct {
uuid [16]byte
slot int
}

func startPairForLockTest(t *testing.T) (*MultiNodeCluster, registeredNode, *syncBuffer) {
t.Helper()
return startPairForLockTestWithEnv(t, nil)
}

// startPairForLockTestWithEnv is startPairForLockTest with envA added to node A's environment.
func startPairForLockTestWithEnv(t *testing.T, envA []string) (*MultiNodeCluster, registeredNode, *syncBuffer) {
t.Helper()
cluster := NewMultiNodeCluster(t, 2, 50)
cluster.Node(0).Env = envA
logA, logB := &syncBuffer{}, &syncBuffer{}
cluster.Node(0).Log, cluster.Node(1).Log = logA, logB
if err := cluster.FormatAndStartFirst(false); err != nil {
Expand Down
9 changes: 9 additions & 0 deletions tests/test-manifest.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -637,6 +637,15 @@ scenarios:
docker: TODO # needs a lock record written as the stopped node at a known moment; Go-only harness
vm: TODO # same as docker

- id: stalled-node-writes-stop-at-write-gate
name: Stalled node's write stops at the write gate
description: Node A is stopped while a writer writes through it; after node B replays A and writes the file, A resumes with its heartbeat held, so its queued write reaches the write gate first, which refuses it and stops A, and the file keeps B's data byte for byte
priority: high
environments:
go: lock_reclaim_test.go::TestStalledNodesWritesStopAtWriteGate
docker: TODO # needs SIGSTOP and SIGCONT on one container's mount process with a writer through it; covered in Go and VM
vm: run-vm-tests.sh::test_stalled_node_writes_stop_at_write_gate

- id: lock-adopt-owner
name: Replaying node takes over a dead node's exclusive locks
description: AdoptOwner holds the dead node's exclusive records through the replay and ReleaseAdopted frees them; shared records and other nodes' records stay
Expand Down
85 changes: 85 additions & 0 deletions tests/vm/scripts/run-vm-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ test_failed=0
run_test() {
local test_name="$1"
local test_func="$2"
# VM_TEST_ONLY runs only the tests whose names contain it
if [[ -n "${VM_TEST_ONLY:-}" && "$test_name" != *"$VM_TEST_ONLY"* ]]; then
return 0
fi

log_info "=========================================="
log_info "Running test: $test_name"
Expand Down Expand Up @@ -2987,6 +2991,86 @@ unmount_goclusterfs() {
done
}

# A node stopped while a writer writes through it is proven dead and replayed by
# a peer, and the peer writes the file. When the stopped node resumes, its
# heartbeat is held (GOCLUSTERFS_TEST_HEARTBEAT_HOLD), so its queued write reaches
# the write gate first. The gate must refuse it and stop the node, and the
# file must keep the peer's data byte for byte.
test_stalled_node_writes_stop_at_write_gate() {
log_info "Testing that a stalled node's queued write stops at the write gate..."
if [[ $NODE_COUNT -lt 2 ]]; then
log_warn "Skipping: Need at least 2 VMs for this test"
return 0
fi
ensure_cluster_healthy "$NODE_COUNT"
local stalled=1 peer=2 pids waited
unmount_goclusterfs "$stalled" || { log_error "VM $stalled: unmount failed"; return 1; }
setup_goclusterfs "$stalled" false "GOCLUSTERFS_TEST_HEARTBEAT_HOLD=30s"

ssh_run "$peer" "head -c 4096 /dev/zero | tr '\\000' 0 > ${GOCLUSTERFS_MOUNT}/fenced && sync"
ssh_run "$stalled" "head -c 4096 /dev/zero | tr '\\000' A > /tmp/fenced.A"
ssh_run "$peer" "head -c 4096 /dev/zero | tr '\\000' B > /tmp/fenced.B"
ssh_run "$stalled" "nohup sh -c 'while dd if=/tmp/fenced.A of=${GOCLUSTERFS_MOUNT}/fenced bs=4096 count=1 conv=notrunc 2>/dev/null; do :; done' < /dev/null > /dev/null 2>&1 &"
sleep 1

local marker="stalled-node gate test: stopping VM $stalled $(date +%s)"
mark_vm_logs "$marker"
pids=$(mount_pids "$stalled")
ssh_run "$stalled" "kill -STOP $(echo $pids)"
local peer_log stalled_log
peer_log=$(get_vm_mount_log "$peer")
stalled_log=$(get_vm_mount_log "$stalled")
waited=0
until sed -n "/$marker/,\$p" "$peer_log" | grep -q "Starting journal replay for node"; do
if [[ $waited -ge 120 ]]; then
log_error "VM $peer did not start replaying VM $stalled within 120s"
ssh_run "$stalled" "kill -CONT $(echo $pids)"
return 1
fi
sleep 1
waited=$((waited + 1))
done
sleep 5
if ! ssh_run "$peer" "dd if=/tmp/fenced.B of=${GOCLUSTERFS_MOUNT}/fenced bs=4096 count=1 conv=notrunc 2>/dev/null && sync"; then
log_error "VM $peer could not write after taking over"
ssh_run "$stalled" "kill -CONT $(echo $pids)"
return 1
fi
ssh_run "$stalled" "kill -CONT $(echo $pids)"

# The held heartbeat stops the node after 30s whatever the gate does; by then a write the gate let through has landed.
waited=0
while [[ -n "$(mount_pids "$stalled")" && $waited -lt 45 ]]; do
sleep 1
waited=$((waited + 1))
done
local result=0
if ! ssh_run "$peer" "cmp -s ${GOCLUSTERFS_MOUNT}/fenced /tmp/fenced.B"; then
log_error "fenced does not hold VM $peer's data after VM $stalled resumed"
result=1
fi
if [[ -n "$(mount_pids "$stalled")" ]]; then
log_error "VM $stalled did not stop after resuming from its takeover"
result=1
fi
if ! sed -n "/$marker/,\$p" "$stalled_log" | grep -q "checking the slot before a write"; then
log_error "VM $stalled stopped without its write gate checking the slot"
result=1
fi

if [[ -n "$(mount_pids "$stalled")" ]]; then
crash_goclusterfs "$stalled" || true
fi
# The bracket keeps pkill from matching the shell that runs it.
ssh_run "$stalled" "pkill -f 'of=${GOCLUSTERFS_MOUNT}/fence[d]'; fusermount3 -uz ${GOCLUSTERFS_MOUNT}" || true
if ! setup_goclusterfs "$stalled" false; then
log_error "VM $stalled: remount after the test failed"
result=1
fi
[[ $result -eq 0 ]] && log_info "Stalled node's queued write stopped at the write gate"
return $result
}

# A power failure while files are written may leave blocks that no file maps.
# With every node unmounted, fsck -repair frees any it finds, the check after
# it is clean, and no file changes.
Expand Down Expand Up @@ -3538,6 +3622,7 @@ main() {
run_test "Reversed Write Order" test_reversed_write_order
run_test "Concurrent Writes Power Failure" test_concurrent_writes_power_failure
run_test "fsck Repair After Power Failure" test_fsck_repair_after_power_failure
run_test "Stalled Node Writes Stop At Write Gate" test_stalled_node_writes_stop_at_write_gate

# Summary
log_info "=========================================="
Expand Down
4 changes: 3 additions & 1 deletion tests/vm/vm-test-harness.sh
Original file line number Diff line number Diff line change
Expand Up @@ -512,6 +512,8 @@ report_boot_failure() {
setup_goclusterfs() {
local vm_id="$1"
local format_disk="${2:-false}"
# Extra environment for the mount process, such as a test failpoint
local mount_env="${3:-}"
local ssh_port=$(get_vm_ssh_port "$vm_id")

log_info "Setting up GoClusterFS on VM $vm_id (format=$format_disk)..."
Expand All @@ -535,7 +537,7 @@ setup_goclusterfs() {
# /var/log/goclusterfs.log is lost when the VM dies; the log device keeps a copy on the host
log_info "Mounting GoClusterFS..."
ssh_run "$vm_id" \
"nohup /usr/local/bin/goclusterfs-mount $mount_args < /dev/null 2>&1 | tee /var/log/goclusterfs.log > /dev/goclusterfs-log 2>/dev/null &"
"nohup env $mount_env /usr/local/bin/goclusterfs-mount $mount_args < /dev/null 2>&1 | tee /var/log/goclusterfs.log > /dev/goclusterfs-log 2>/dev/null &"

# Wait for mount to be ready
wait_for_goclusterfs_mount "$vm_id"
Expand Down
Loading