From 330b73e129934821f37076241127f0c29282879b Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 09:15:27 -0600 Subject: [PATCH 1/8] test: add a multi-node soak driver with a model, faults and replay goclusterfs-soak runs a random mix of operations from several lanes on a 3-VM cluster and checks every node against a model within the coherence window. It cuts power, SIGKILLs or stops nodes past the heartbeat timeout, runs fsck from a quiesced cluster every hour, logs every operation, and replays a log in order. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 2 + Makefile | 6 +- tests/soak/README.md | 36 +++ tests/soak/agent.go | 174 +++++++++++ tests/soak/cluster.go | 189 ++++++++++++ tests/soak/cluster.sh | 129 ++++++++ tests/soak/driver.go | 629 +++++++++++++++++++++++++++++++++++++++ tests/soak/main.go | 126 ++++++++ tests/soak/model.go | 428 ++++++++++++++++++++++++++ tests/soak/model_test.go | 70 +++++ 10 files changed, 1788 insertions(+), 1 deletion(-) create mode 100644 tests/soak/README.md create mode 100644 tests/soak/agent.go create mode 100644 tests/soak/cluster.go create mode 100644 tests/soak/cluster.sh create mode 100644 tests/soak/driver.go create mode 100644 tests/soak/main.go create mode 100644 tests/soak/model.go create mode 100644 tests/soak/model_test.go diff --git a/.gitignore b/.gitignore index cd338222..7605297a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ /goclusterfs /goclusterfs-mount /goclusterfs-fsck +/goclusterfs-soak +/soak-out/ /gfsctl /GoClusterFS /coverage.out diff --git a/Makefile b/Makefile index f987cd46..85b2981e 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # GoClusterFS Makefile # Build and test automation for the distributed filesystem -.PHONY: build build-mount build-fsck build-gfsctl completions dashboard test test-unit test-integration test-all test-parity test-multinode-docker docker-build-test docker-clean-test test-multinode-vm vm-clean clean fmt vet check-fuse check-qemu lint vuln help +.PHONY: build build-mount build-fsck build-gfsctl soak completions dashboard test test-unit test-integration test-all test-parity test-multinode-docker docker-build-test docker-clean-test test-multinode-vm vm-clean clean fmt vet check-fuse check-qemu lint vuln help # Go parameters GOCMD=go @@ -41,6 +41,10 @@ build-fsck: $(GOBUILD) -o $(FSCK_BINARY) ./cmd/goclusterfs-fsck @echo "Built $(FSCK_BINARY) successfully" +# Static, so the same binary runs on the host as the driver and in the Alpine VMs as the agent +soak: build-mount build-fsck + CGO_ENABLED=0 $(GOBUILD) -o goclusterfs-soak ./tests/soak + build-gfsctl: @echo "Building $(GFSCTL_BINARY)..." $(GOBUILD) -o $(GFSCTL_BINARY) ./cmd/gfsctl diff --git a/tests/soak/README.md b/tests/soak/README.md new file mode 100644 index 00000000..84fc7071 --- /dev/null +++ b/tests/soak/README.md @@ -0,0 +1,36 @@ +# Multi-node soak + +`goclusterfs-soak` runs a random mix of operations on a 3-VM cluster for hours or days and checks every node against a model of the expected state. + +- **Operations:** create, write (overwrite, append, sparse), truncate, unlink, mkdir, rmdir, rename, snapshot, clone, clone from a snapshot, and snapshot delete. Rename is not supported, so each rename must fail and change nothing. +- **Lanes:** several lanes run at once, each on its own files and snapshots, in directories they share. +- **Checks:** after every operation, each node must show its result, through a fresh open, within `-window` (10s). That covers `-attr-timeout` and `-cache-ttl`, because broadcast cannot cross the VMs' separate networks. +- **Faults:** every `-fault-every`, one node is hit, picked at random from these: + - power is cut (QEMU is killed); + - goclusterfs-mount gets SIGKILL; + - goclusterfs-mount is stopped past the 10s heartbeat timeout and resumed. + + The node comes back, mounts again and must show the model. The other nodes keep working through the fault. +- **fsck:** every `-fsck-every`, all operations stop, every node unmounts and goclusterfs-fsck runs. Anything other than leaks and orphan snapshots is a failure. Leaks are repaired. + +The first failure stops the run. `result.txt` says what failed, `ops.jsonl` logs every operation, fault and fsck, and `cluster.img` keeps the shared disk. + +## Run + +On a host with KVM, from a checkout of its own: + +```bash +make soak +flock /tmp/goclusterfs-soak.lock systemd-run --scope -p MemoryMax=8G -p CPUQuota=400% --quiet \ + ./goclusterfs-soak run -duration 48h -out /root/gcfs-soak/run1 +``` + +The VMs use SSH ports from `-ssh-base-port` (3122), so the soak can share a host with other VM test runs. It logs a summary every minute. + +## Replay + +```bash +./goclusterfs-soak replay -log /root/gcfs-soak/run1/ops.jsonl -out /root/gcfs-soak/replay1 +``` + +Replay boots a new cluster and runs the logged operations one at a time, in log order, on the same nodes, with the same data, faults and fsck runs. `-upto N` stops after operation N. Lanes ran at the same time in the original run, so a failure that needs their exact timing may not happen again. diff --git a/tests/soak/agent.go b/tests/soak/agent.go new file mode 100644 index 00000000..49acc71f --- /dev/null +++ b/tests/soak/agent.go @@ -0,0 +1,174 @@ +//go:build linux + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand/v2" + "os" + "path/filepath" + "sync" + "syscall" + + "golang.org/x/sys/unix" +) + +// request is one operation the driver asks an agent to run. Paths are relative to the mount. +type request struct { + ID uint64 `json:"id"` + Op string `json:"op"` + Path string `json:"path,omitempty"` + Path2 string `json:"path2,omitempty"` + Attr string `json:"attr,omitempty"` + Value string `json:"value,omitempty"` + Off int64 `json:"off,omitempty"` + Len int64 `json:"len,omitempty"` + Seed uint64 `json:"seed,omitempty"` +} + +type response struct { + ID uint64 `json:"id"` + Errno int `json:"errno"` + Err string `json:"err,omitempty"` + Size int64 `json:"size,omitempty"` + Sum string `json:"sum,omitempty"` + Names []string `json:"names,omitempty"` + Text string `json:"text,omitempty"` +} + +// genData returns n bytes that depend only on seed, so the driver and the agents produce the same data. +func genData(seed uint64, n int64) []byte { + b := make([]byte, n) + r := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)) + for i := int64(0); i < n; i += 8 { + v := r.Uint64() + for j := int64(0); j < 8 && i+j < n; j++ { + b[i+j] = byte(v >> (8 * j)) + } + } + return b +} + +func sum(b []byte) string { + h := sha256.Sum256(b) + return hex.EncodeToString(h[:8]) +} + +// runAgent serves requests from stdin until it closes, running each in its own goroutine. +func runAgent(root string, in io.Reader, out io.Writer) error { + dec := json.NewDecoder(in) + enc := json.NewEncoder(out) + var mu sync.Mutex + var wg sync.WaitGroup + defer wg.Wait() + for { + var req request + if err := dec.Decode(&req); err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + wg.Add(1) + go func() { + defer wg.Done() + resp := serve(root, req) + resp.ID = req.ID + mu.Lock() + defer mu.Unlock() + if err := enc.Encode(resp); err != nil { + fmt.Fprintf(os.Stderr, "agent: reply %d: %v\n", req.ID, err) + } + }() + } +} + +func serve(root string, req request) response { + var resp response + p := filepath.Join(root, req.Path) + var err error + switch req.Op { + case "ping": + case "create": + var f *os.File + if f, err = os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644); err == nil { + err = f.Close() + } + case "write": + err = writeSynced(p, req.Off, genData(req.Seed, req.Len)) + case "truncate": + err = unix.Truncate(p, req.Len) + case "unlink": + err = unix.Unlink(p) + case "mkdir": + err = unix.Mkdir(p, 0o755) + case "rmdir": + err = unix.Rmdir(p) + case "rename": + err = unix.Rename(p, filepath.Join(root, req.Path2)) + case "setxattr": + err = unix.Setxattr(p, req.Attr, []byte(req.Value), 0) + case "getxattr": + buf := make([]byte, 64<<10) + var n int + if n, err = unix.Getxattr(p, req.Attr, buf); err == nil { + resp.Text = string(buf[:n]) + } + case "read": + var data []byte + if data, err = os.ReadFile(p); err == nil { + resp.Size, resp.Sum = int64(len(data)), sum(data) + } + case "list": + var entries []os.DirEntry + if entries, err = os.ReadDir(p); err == nil { + for _, e := range entries { + resp.Names = append(resp.Names, entryName(p, e)) + } + } + default: + err = fmt.Errorf("unknown op %q", req.Op) + } + if err != nil { + resp.Err = err.Error() + resp.Errno = -1 + var errno syscall.Errno + if errors.As(err, &errno) { + resp.Errno = int(errno) + } + } + return resp +} + +// entryName describes a directory entry as "d:name" or "f:name:size". +func entryName(dir string, e os.DirEntry) string { + if e.IsDir() { + return "d:" + e.Name() + } + var st unix.Stat_t + if err := unix.Lstat(filepath.Join(dir, e.Name()), &st); err != nil { + return "f:" + e.Name() + ":?" + } + return fmt.Sprintf("f:%s:%d", e.Name(), st.Size) +} + +func writeSynced(p string, off int64, data []byte) error { + f, err := os.OpenFile(p, os.O_WRONLY, 0) + if err != nil { + return err + } + if _, err := f.WriteAt(data, off); err != nil { + _ = f.Close() + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + return f.Close() +} diff --git a/tests/soak/cluster.go b/tests/soak/cluster.go new file mode 100644 index 00000000..afa4b9c7 --- /dev/null +++ b/tests/soak/cluster.go @@ -0,0 +1,189 @@ +//go:build linux + +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + "sync" + "sync/atomic" +) + +// controller drives tests/soak/cluster.sh, which owns the VMs. +type controller struct { + mu sync.Mutex + cmd *exec.Cmd + in io.WriteCloser + out *bufio.Scanner +} + +func startController(script string, env []string, logw io.Writer) (*controller, error) { + cmd := exec.Command("bash", script) + cmd.Env = append(os.Environ(), env...) + cmd.Stderr = logw + in, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + out, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + sc := bufio.NewScanner(out) + sc.Buffer(make([]byte, 1<<20), 1<<20) + return &controller{cmd: cmd, in: in, out: sc}, nil +} + +// run sends one command and returns the text after "ok" and any "out" lines before it. +func (c *controller) run(command string) (string, []string, error) { + c.mu.Lock() + defer c.mu.Unlock() + if _, err := fmt.Fprintln(c.in, command); err != nil { + return "", nil, err + } + var lines []string + for c.out.Scan() { + line := c.out.Text() + switch { + case strings.HasPrefix(line, "out "): + lines = append(lines, strings.TrimPrefix(line, "out ")) + case line == "ok" || strings.HasPrefix(line, "ok "): + return strings.TrimSpace(strings.TrimPrefix(line, "ok")), lines, nil + case strings.HasPrefix(line, "err"): + return "", lines, fmt.Errorf("controller: %s", line) + } + } + return "", lines, fmt.Errorf("controller exited during %q: %v", command, c.out.Err()) +} + +func (c *controller) stop() { + _, _, _ = c.run("down") + _ = c.in.Close() + _ = c.cmd.Wait() +} + +func sshArgs(key string) []string { + return []string{ + "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR", + "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=6", + "-i", key, + } +} + +// pushAgent copies this binary to a VM, whose root file system does not survive a reboot. +func pushAgent(key string, port int) error { + exe, err := os.Executable() + if err != nil { + return err + } + args := append(sshArgs(key), "-P", fmt.Sprint(port), exe, "root@127.0.0.1:/usr/local/bin/goclusterfs-soak") + if out, err := exec.Command("scp", args...).CombinedOutput(); err != nil { + return fmt.Errorf("scp to port %d: %v: %s", port, err, out) + } + return nil +} + +// agentConn is an ssh session to the agent on one VM. Calls may overlap. +type agentConn struct { + mu sync.Mutex + enc *json.Encoder + pending map[uint64]chan response + nextID atomic.Uint64 + cmd *exec.Cmd + stdin io.WriteCloser + done chan struct{} + err error +} + +var errAgentGone = errors.New("agent connection closed") + +func dialAgent(key string, port int, mount string, logw io.Writer) (*agentConn, error) { + args := append(sshArgs(key), "-p", fmt.Sprint(port), "root@127.0.0.1", + "/usr/local/bin/goclusterfs-soak", "agent", "-root", mount) + cmd := exec.Command("ssh", args...) + cmd.Stderr = logw + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + a := &agentConn{enc: json.NewEncoder(stdin), pending: map[uint64]chan response{}, cmd: cmd, stdin: stdin, done: make(chan struct{})} + go a.read(stdout) + if _, err := a.call(context.Background(), request{Op: "ping"}); err != nil { + a.close() + return nil, fmt.Errorf("agent on port %d: %w", port, err) + } + return a, nil +} + +func (a *agentConn) read(r io.Reader) { + dec := json.NewDecoder(r) + for { + var resp response + if err := dec.Decode(&resp); err != nil { + a.mu.Lock() + a.err = fmt.Errorf("%w: %v", errAgentGone, err) + for id, ch := range a.pending { + close(ch) + delete(a.pending, id) + } + a.mu.Unlock() + close(a.done) + return + } + a.mu.Lock() + ch := a.pending[resp.ID] + delete(a.pending, resp.ID) + a.mu.Unlock() + if ch != nil { + ch <- resp + } + } +} + +func (a *agentConn) call(ctx context.Context, req request) (response, error) { + req.ID = a.nextID.Add(1) + ch := make(chan response, 1) + a.mu.Lock() + if a.err != nil { + a.mu.Unlock() + return response{}, a.err + } + a.pending[req.ID] = ch + err := a.enc.Encode(req) + a.mu.Unlock() + if err != nil { + return response{}, fmt.Errorf("%w: %v", errAgentGone, err) + } + select { + case resp, ok := <-ch: + if !ok { + return response{}, a.err + } + return resp, nil + case <-ctx.Done(): + return response{}, ctx.Err() + } +} + +func (a *agentConn) close() { + _ = a.stdin.Close() + _ = a.cmd.Process.Kill() + _ = a.cmd.Wait() +} diff --git a/tests/soak/cluster.sh b/tests/soak/cluster.sh new file mode 100644 index 00000000..c9dfff7b --- /dev/null +++ b/tests/soak/cluster.sh @@ -0,0 +1,129 @@ +#!/bin/bash +# Cluster controller for the soak driver. It owns the VMs, which die with it, and +# answers one command per stdin line with one "ok ..." or "err ..." line on stdout. +# Everything else goes to stderr. +# +# up boot NODE_COUNT VMs, format and mount; answers "ok ..." +# power VM cut power to VM, boot it again and mount +# sigkill VM SIGKILL VM's goclusterfs-mount and mount again +# stop VM SECONDS SIGSTOP VM's goclusterfs-mount for SECONDS, resume it, then mount again +# remount VM kill VM's goclusterfs-mount and mount again +# unmount-all unmount every node +# mount-all mount every node that is not mounted +# fsck [-repair] run goclusterfs-fsck on VM 1 with every node unmounted; answers +# "out " for each problem line, then "ok rc=" +# save-image DEST copy the shared disk to DEST +# down stop every VM and exit + +set -uo pipefail + +SOAK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +NODE_COUNT=${NODE_COUNT:-3} +source "${SOAK_DIR}/../vm/vm-test-harness.sh" >&2 + +reply() { echo "$*"; } + +mount_pid() { + ssh_run "$1" "pidof goclusterfs-mount" 2>/dev/null +} + +# Kills a node's mount process and clears its mount point, as a crash leaves it. +kill_mount() { + ssh_run "$1" "pkill -9 -x goclusterfs-mount; sleep 1; fusermount3 -uz ${GOCLUSTERFS_MOUNT} 2>/dev/null; true" +} + +remount() { + kill_mount "$1" && setup_goclusterfs "$1" false +} + +unmount_node() { + local vm_id=$1 waited=0 + ssh_run "$vm_id" "fusermount3 -u ${GOCLUSTERFS_MOUNT}" || true + while ssh_run "$vm_id" "pidof goclusterfs-mount > /dev/null"; do + if [[ $waited -ge 30 ]]; then + log_warn "VM $vm_id: goclusterfs-mount still running 30s after unmount, killing it" + kill_mount "$vm_id" + return 0 + fi + sleep 1 + waited=$((waited + 1)) + done +} + +handle() { + local cmd=$1 + shift + case "$cmd" in + up) + create_shared_disk >&2 || return 1 + local vm_id ports=() + for vm_id in $(seq 1 "$NODE_COUNT"); do + start_vm "$vm_id" "$([[ $vm_id -eq 1 ]] && echo true || echo false)" >&2 || return 1 + done + for vm_id in $(seq 1 "$NODE_COUNT"); do + wait_vm_healthy "$vm_id" >&2 || return 1 + setup_goclusterfs "$vm_id" "$([[ $vm_id -eq 1 ]] && echo true || echo false)" >&2 || return 1 + ports+=("$(get_vm_ssh_port "$vm_id")") + done + reply "ok $(get_ssh_key) ${ports[*]}" + ;; + power) + kill_vm "$1" >&2 && start_vm "$1" false >&2 && wait_vm_healthy "$1" >&2 && + setup_goclusterfs "$1" false >&2 && reply ok + ;; + sigkill) + remount "$1" >&2 && reply ok + ;; + stop) + local pid + pid=$(mount_pid "$1") || return 1 + ssh_run "$1" "kill -STOP $pid" >&2 || return 1 + sleep "$2" + ssh_run "$1" "kill -CONT $pid" >&2 + # The other nodes declared it failed and fenced it, so it has to mount again. + remount "$1" >&2 && reply ok + ;; + remount) + remount "$1" >&2 && reply ok + ;; + unmount-all) + local vm_id + for vm_id in $(seq 1 "$NODE_COUNT"); do + unmount_node "$vm_id" >&2 + done + reply ok + ;; + mount-all) + local vm_id + for vm_id in $(seq 1 "$NODE_COUNT"); do + if ! ssh_run "$vm_id" "mountpoint -q ${GOCLUSTERFS_MOUNT}"; then + setup_goclusterfs "$vm_id" false >&2 || return 1 + fi + done + reply ok + ;; + fsck) + local out rc + out=$(ssh_run 1 "/usr/local/bin/goclusterfs-fsck ${1:-} /dev/vda 2>&1; echo rc=\$?") + rc=${out##*rc=} + echo "$out" | grep -E '^ *\[(WARN|ERROR)\]' | sed 's/^ */out /' + reply "ok rc=$rc" + ;; + save-image) + cp --sparse=always "${SHARED_DISK_PATH}" "$1" >&2 && reply ok + ;; + down) + reply ok + exit 0 + ;; + *) + reply "err unknown command $cmd" + return 0 + ;; + esac +} + +while read -r line; do + # shellcheck disable=SC2086 + handle $line || reply "err $line failed" +done diff --git a/tests/soak/driver.go b/tests/soak/driver.go new file mode 100644 index 00000000..9652cdd8 --- /dev/null +++ b/tests/soak/driver.go @@ -0,0 +1,629 @@ +//go:build linux + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "math/rand/v2" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" +) + +const mountPoint = "/mnt/goclusterfs" + +type config struct { + checkout string + out string + duration time.Duration + lanes int + seed uint64 + faultEvery time.Duration + fsckEvery time.Duration + barrierEvery time.Duration + window time.Duration + stopFor time.Duration + opTimeout time.Duration + sshBasePort int + diskMB int + budget int64 +} + +type node struct { + id int + port int + mu sync.RWMutex // an operation holds it shared; a fault holds it exclusively + up atomic.Bool + agent *agentConn +} + +// record is one line of the operation log. +type record struct { + Type string `json:"type"` + Op + Fault string `json:"fault,omitempty"` + Victim int `json:"victim,omitempty"` + Detail string `json:"detail,omitempty"` +} + +type driver struct { + cfg config + ctl *controller + key string + nodes []*node + model *Model + pause sync.RWMutex // lanes hold it shared per operation; barriers and fsck hold it exclusively + faultMu sync.Mutex + seq atomic.Uint64 + logMu sync.Mutex + logw io.Writer + stderr io.Writer + ctx context.Context + cancel context.CancelFunc + failMu sync.Mutex + failure string + + ops, unexpected, faults, fscks, barriers atomic.Int64 + slowestMs atomic.Int64 + opsByKind sync.Map +} + +func (d *driver) logRecord(r record) { + d.logMu.Lock() + defer d.logMu.Unlock() + b, err := json.Marshal(r) + if err == nil { + _, err = d.logw.Write(append(b, '\n')) + } + if err != nil { + log.Printf("op log: %v", err) + } +} + +// fail stops the soak at the first failure and keeps its description. +func (d *driver) fail(format string, args ...any) { + d.failMu.Lock() + defer d.failMu.Unlock() + if d.failure != "" { + return + } + d.failure = fmt.Sprintf(format, args...) + log.Printf("FAILURE: %s", d.failure) + d.logRecord(record{Type: "failure", Detail: d.failure}) + d.cancel() +} + +func (d *driver) bringUp() error { + reply, _, err := d.ctl.run("up") + if err != nil { + return err + } + fields := strings.Fields(reply) + if len(fields) < 2 { + return fmt.Errorf("controller answered up with %q", reply) + } + d.key = fields[0] + for i, p := range fields[1:] { + n := &node{id: i + 1} + if _, err := fmt.Sscan(p, &n.port); err != nil { + return err + } + d.nodes = append(d.nodes, n) + } + for _, n := range d.nodes { + if err := d.connect(n); err != nil { + return err + } + } + for _, dir := range append([]string{"/soak"}, sharedDirs...) { + if resp, err := d.nodes[0].agent.call(d.ctx, request{Op: "mkdir", Path: dir}); err != nil || resp.Errno != 0 { + return fmt.Errorf("mkdir %s: %v %s", dir, err, resp.Err) + } + } + return nil +} + +func (d *driver) connect(n *node) error { + if err := pushAgent(d.key, n.port); err != nil { + return err + } + a, err := dialAgent(d.key, n.port, mountPoint, d.stderr) + if err != nil { + return err + } + n.agent = a + n.up.Store(true) + return nil +} + +func (d *driver) upNodes() []*node { + var up []*node + for _, n := range d.nodes { + if n.up.Load() { + up = append(up, n) + } + } + return up +} + +// converge retries check until it passes or the coherence window ends. The caller holds the node. +func (d *driver) converge(check func() error) error { + start := time.Now() + for { + err := check() + if err == nil { + if ms := time.Since(start).Milliseconds(); ms > d.slowestMs.Load() { + d.slowestMs.Store(ms) + } + return nil + } + if errors.Is(err, errAgentGone) || time.Since(start) > d.cfg.window || d.ctx.Err() != nil { + return fmt.Errorf("%v (still after %s)", err, time.Since(start).Round(time.Millisecond)) + } + time.Sleep(100 * time.Millisecond) + } +} + +func (d *driver) call(n *node, req request) (response, error) { + ctx, cancel := context.WithTimeout(d.ctx, d.cfg.opTimeout) + defer cancel() + return n.agent.call(ctx, req) +} + +// comparePath checks what node n shows at p against the model. +func (d *driver) comparePath(n *node, p string) error { + d.model.mu.Lock() + f, isDir := d.model.Files[p], d.model.Dirs[p] + var size int64 + var want string + if f != nil { + size, want = int64(len(f.Data)), sum(f.Data) + } + d.model.mu.Unlock() + switch { + case f != nil: + resp, err := d.call(n, request{Op: "read", Path: p}) + if err != nil { + return err + } + if resp.Errno != 0 || resp.Size != size || resp.Sum != want { + return fmt.Errorf("%s: read errno %d size %d sum %s, want size %d sum %s", p, resp.Errno, resp.Size, resp.Sum, size, want) + } + case isDir: + resp, err := d.call(n, request{Op: "list", Path: p}) + if err != nil { + return err + } + if resp.Errno != 0 { + return fmt.Errorf("%s: list errno %d (%s), want a directory", p, resp.Errno, resp.Err) + } + default: + resp, err := d.call(n, request{Op: "read", Path: p}) + if err != nil { + return err + } + if resp.Errno != int(syscall.ENOENT) { + return fmt.Errorf("%s: read errno %d size %d, want ENOENT", p, resp.Errno, resp.Size) + } + } + return nil +} + +func (d *driver) compareSnaps(n *node, p string) error { + d.model.mu.Lock() + want, exists := d.model.snapsOf(p), d.model.Files[p] != nil + d.model.mu.Unlock() + if !exists { + return nil + } + resp, err := d.call(n, request{Op: "getxattr", Path: p, Attr: "user.snapshots"}) + if err != nil { + return err + } + var got []string + for _, line := range strings.FieldsFunc(resp.Text, func(r rune) bool { return r == '\n' || r == ',' }) { + if name, state, ok := strings.Cut(line, ":"); ok && state == "active" { + got = append(got, name) + } + } + sort.Strings(got) + if resp.Errno != 0 || !slices.Equal(got, want) { + return fmt.Errorf("%s: snapshots %v (errno %d), want %v", p, got, resp.Errno, want) + } + return nil +} + +func (d *driver) compareDir(n *node, dir string) error { + d.model.mu.Lock() + want := d.model.listing(dir) + d.model.mu.Unlock() + resp, err := d.call(n, request{Op: "list", Path: dir}) + if err != nil { + return err + } + got := slices.Clone(resp.Names) + sort.Strings(got) + if resp.Errno != 0 || !slices.Equal(got, want) { + return fmt.Errorf("%s: lists %v (errno %d), want %v", dir, got, resp.Errno, want) + } + return nil +} + +// verifyAll checks every directory listing, and with content every file and its snapshots, on node n. +func (d *driver) verifyAll(n *node, content bool) error { + d.model.mu.Lock() + var dirs, files []string + for p := range d.model.Dirs { + dirs = append(dirs, p) + } + for p := range d.model.Files { + files = append(files, p) + } + d.model.mu.Unlock() + sort.Strings(dirs) + sort.Strings(files) + for _, dir := range dirs { + if err := d.converge(func() error { return d.compareDir(n, dir) }); err != nil { + return err + } + } + if !content { + return nil + } + for _, p := range files { + if err := d.converge(func() error { return d.comparePath(n, p) }); err != nil { + return err + } + if err := d.converge(func() error { return d.compareSnaps(n, p) }); err != nil { + return err + } + } + return nil +} + +func (d *driver) runLane(l *lane) { + for d.ctx.Err() == nil { + d.pause.RLock() + d.model.mu.Lock() + op := l.generate(d.model) + op.Seq = d.seq.Add(1) + op.Want = d.model.expect(op) + d.model.mu.Unlock() + for d.ctx.Err() == nil { + up := d.upNodes() + if len(up) == 0 { + time.Sleep(100 * time.Millisecond) + continue + } + n := pick(l.rng, up) + n.mu.RLock() + if !n.up.Load() { + n.mu.RUnlock() + continue + } + d.runOpLocked(op, n) + break + } + d.pause.RUnlock() + } +} + +// runOpLocked runs op on n, which the caller holds shared, and releases n before checking the other nodes. +func (d *driver) runOpLocked(op *Op, n *node) { + op.Node = n.id + op.At = time.Now().UnixMilli() + ctx, cancel := context.WithTimeout(d.ctx, d.cfg.opTimeout) + resp, err := n.agent.call(ctx, op.request()) + cancel() + n.mu.RUnlock() + d.finishOp(op, resp, err) +} + +func (d *driver) finishOp(op *Op, resp response, err error) { + op.Ms = time.Now().UnixMilli() - op.At + if err != nil { + if d.ctx.Err() == nil { + d.fail("op %d %s %s on node %d: %v", op.Seq, op.Kind, op.Path, op.Node, err) + } + return + } + op.Got, op.Err = resp.Errno, resp.Err + d.check(op) +} + +// check logs a finished operation, compares its errno with the model, applies it and checks every node shows it. +func (d *driver) check(op *Op) { + d.logRecord(record{Type: "op", Op: *op}) + d.ops.Add(1) + count, _ := d.opsByKind.LoadOrStore(op.Kind, new(atomic.Int64)) + count.(*atomic.Int64).Add(1) + if !(op.Want == anyErrno && op.Got != 0 || op.Want == op.Got) { + d.unexpected.Add(1) + d.fail("op %d %s %s on node %d: errno %d (%s), want %d", op.Seq, op.Kind, op.Path, op.Node, op.Got, op.Err, op.Want) + return + } + if op.Got == 0 { + d.model.mu.Lock() + d.model.apply(op) + d.model.mu.Unlock() + } + paths := []string{op.Path} + if op.Path2 != "" { + paths = append(paths, op.Path2) + } + for _, m := range d.upNodes() { + if err := d.onNode(m, func() error { return d.checkOp(m, op, paths) }); err != nil { + d.fail("after op %d %s %s on node %d: node %d: %v", op.Seq, op.Kind, op.Path, op.Node, m.id, err) + return + } + } +} + +func (d *driver) checkOp(m *node, op *Op, paths []string) error { + for _, p := range paths { + if err := d.converge(func() error { return d.comparePath(m, p) }); err != nil { + return err + } + } + if slices.Contains([]string{"snapshot", "snapdelete", "snapclone"}, op.Kind) { + return d.converge(func() error { return d.compareSnaps(m, op.Path) }) + } + return nil +} + +// onNode runs f while holding node n shared, unless a fault has taken n down. +func (d *driver) onNode(n *node, f func() error) error { + if !n.up.Load() { + return nil + } + n.mu.RLock() + defer n.mu.RUnlock() + if !n.up.Load() { + return nil + } + return f() +} + +// injectFault takes one node down, brings it back and checks it shows the model. Operations go on elsewhere. +func (d *driver) injectFault(kind string, n *node) { + d.faultMu.Lock() + defer d.faultMu.Unlock() + start := time.Now() + n.up.Store(false) + n.mu.Lock() + defer n.mu.Unlock() + n.agent.close() + command := fmt.Sprintf("%s %d", kind, n.id) + if kind == "stop" { + command = fmt.Sprintf("stop %d %d", n.id, int(d.cfg.stopFor.Seconds())) + } + log.Printf("fault: %s", command) + if _, _, err := d.ctl.run(command); err != nil { + d.fail("fault %s: %v", command, err) + return + } + if err := d.connect(n); err != nil { + d.fail("fault %s: reconnect: %v", command, err) + return + } + n.up.Store(false) + d.faults.Add(1) + d.logRecord(record{Type: "fault", Fault: kind, Victim: n.id, Op: Op{At: start.UnixMilli(), Ms: time.Since(start).Milliseconds()}}) + if err := d.verifyAll(n, true); err != nil { + d.fail("after fault %s: node %d: %v", command, n.id, err) + return + } + n.up.Store(true) +} + +// quiescedFsck stops all operations, unmounts every node, runs fsck, and mounts again. +func (d *driver) quiescedFsck() { + d.pause.Lock() + defer d.pause.Unlock() + d.faultMu.Lock() + defer d.faultMu.Unlock() + start := time.Now() + for _, n := range d.nodes { + n.up.Store(false) + n.agent.close() + } + if _, _, err := d.ctl.run("unmount-all"); err != nil { + d.fail("fsck: unmount: %v", err) + return + } + rc, lines, err := d.ctl.run("fsck") + if err != nil { + d.fail("fsck: %v", err) + return + } + var leaks, other []string + for _, line := range lines { + switch { + case strings.Contains(line, "is an orphan"): + case isLeak(line): + leaks = append(leaks, line) + default: + other = append(other, line) + } + } + d.fscks.Add(1) + d.logRecord(record{Type: "fsck", Detail: fmt.Sprintf("%s, %d leaks, %d other", rc, len(leaks), len(other)), Op: Op{At: start.UnixMilli()}}) + if len(other) > 0 { + d.fail("fsck found more than leaks:\n%s", strings.Join(other, "\n")) + return + } + if len(leaks) > 0 { + log.Printf("fsck: repairing %d leaks", len(leaks)) + if _, _, err := d.ctl.run("fsck -repair"); err != nil { + d.fail("fsck -repair: %v", err) + return + } + if _, lines, err = d.ctl.run("fsck"); err != nil || slices.ContainsFunc(lines, func(l string) bool { return !strings.Contains(l, "is an orphan") }) { + d.fail("fsck after repair: %v %v", err, lines) + return + } + } + if _, _, err := d.ctl.run("mount-all"); err != nil { + d.fail("fsck: mount: %v", err) + return + } + for _, n := range d.nodes { + if err := d.connect(n); err != nil { + d.fail("fsck: reconnect node %d: %v", n.id, err) + return + } + } + for _, n := range d.nodes { + if err := d.verifyAll(n, true); err != nil { + d.fail("after fsck: node %d: %v", n.id, err) + return + } + } + log.Printf("fsck: %d leaks repaired, every node shows the model (%s)", len(leaks), time.Since(start).Round(time.Second)) +} + +func isLeak(line string) bool { + for _, s := range []string{"leaked", "will never be freed", "unnamed: no directory names it", "belongs to no snapshot"} { + if strings.Contains(line, s) { + return true + } + } + return false +} + +// barrier pauses every lane and checks every node lists every directory as the model does. +func (d *driver) barrier() { + d.pause.Lock() + defer d.pause.Unlock() + for _, n := range d.upNodes() { + if err := d.onNode(n, func() error { return d.verifyAll(n, false) }); err != nil { + d.fail("barrier: node %d: %v", n.id, err) + return + } + } + d.barriers.Add(1) +} + +func (d *driver) report() string { + d.model.mu.Lock() + files, snaps, blocks := len(d.model.Files), len(d.model.Snaps), d.model.blocks() + d.model.mu.Unlock() + var kinds []string + d.opsByKind.Range(func(k, v any) bool { + kinds = append(kinds, fmt.Sprintf("%s=%d", k, v.(*atomic.Int64).Load())) + return true + }) + sort.Strings(kinds) + return fmt.Sprintf("ops=%d unexpected=%d faults=%d fscks=%d barriers=%d slowest_converge_ms=%d files=%d snapshots=%d blocks=%d [%s]", + d.ops.Load(), d.unexpected.Load(), d.faults.Load(), d.fscks.Load(), d.barriers.Load(), d.slowestMs.Load(), + files, snaps, blocks, strings.Join(kinds, " ")) +} + +func newDriver(cfg config) (*driver, func(), error) { + if err := os.MkdirAll(cfg.out, 0o755); err != nil { + return nil, nil, err + } + logf, err := os.Create(filepath.Join(cfg.out, "ops.jsonl")) + if err != nil { + return nil, nil, err + } + clusterLog, err := os.Create(filepath.Join(cfg.out, "cluster.log")) + if err != nil { + return nil, nil, err + } + env := []string{"NODE_COUNT=3", fmt.Sprintf("VM_SSH_BASE_PORT=%d", cfg.sshBasePort), fmt.Sprintf("VM_CLUSTER_SIZE_MB=%d", cfg.diskMB)} + ctl, err := startController(filepath.Join(cfg.checkout, "tests/soak/cluster.sh"), env, clusterLog) + if err != nil { + return nil, nil, err + } + ctx, cancel := context.WithCancel(context.Background()) + d := &driver{cfg: cfg, ctl: ctl, model: newModel(), logw: logf, stderr: clusterLog, ctx: ctx, cancel: cancel} + cleanup := func() { + cancel() + for _, n := range d.nodes { + if n.agent != nil { + n.agent.close() + } + } + ctl.stop() + _ = logf.Close() + _ = clusterLog.Close() + } + return d, cleanup, nil +} + +func every(ctx context.Context, interval time.Duration, f func()) { + if interval <= 0 { + return + } + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + f() + } + } +} + +func run(cfg config) error { + d, cleanup, err := newDriver(cfg) + if err != nil { + return err + } + defer cleanup() + log.Printf("soak: seed %d, %d lanes, %s, faults every %s, fsck every %s, output in %s", + cfg.seed, cfg.lanes, cfg.duration, cfg.faultEvery, cfg.fsckEvery, cfg.out) + if err := d.bringUp(); err != nil { + return fmt.Errorf("bring up the cluster: %w", err) + } + ctx, stop := context.WithTimeout(d.ctx, cfg.duration) + defer stop() + faultRng := rand.New(rand.NewPCG(cfg.seed, 0xfa)) + var wg sync.WaitGroup + for i := range cfg.lanes { + l := &lane{id: i, rng: rand.New(rand.NewPCG(cfg.seed, uint64(i))), budget: cfg.budget} + wg.Go(func() { d.runLane(l) }) + } + wg.Go(func() { + every(ctx, cfg.faultEvery, func() { + d.injectFault(pick(faultRng, []string{"power", "sigkill", "stop"}), pick(faultRng, d.nodes)) + }) + }) + wg.Go(func() { every(ctx, cfg.fsckEvery, d.quiescedFsck) }) + wg.Go(func() { every(ctx, cfg.barrierEvery, d.barrier) }) + wg.Go(func() { every(ctx, time.Minute, func() { log.Print(d.report()) }) }) + <-ctx.Done() + d.cancel() + wg.Wait() + log.Print(d.report()) + return d.finish() +} + +// finish writes the outcome and, after a failure, keeps the shared disk for analysis. +func (d *driver) finish() error { + if d.failure == "" { + return os.WriteFile(filepath.Join(d.cfg.out, "result.txt"), []byte("PASS "+d.report()+"\n"), 0o644) + } + text := "FAIL " + d.failure + "\n" + d.report() + "\n" + if err := os.WriteFile(filepath.Join(d.cfg.out, "result.txt"), []byte(text), 0o644); err != nil { + return err + } + if _, _, err := d.ctl.run("save-image " + filepath.Join(d.cfg.out, "cluster.img")); err != nil { + log.Printf("save the shared disk: %v", err) + } + return errors.New(d.failure) +} diff --git a/tests/soak/main.go b/tests/soak/main.go new file mode 100644 index 00000000..e175d3b4 --- /dev/null +++ b/tests/soak/main.go @@ -0,0 +1,126 @@ +//go:build linux + +// Command goclusterfs-soak runs a long random workload on a 3-VM cluster, checks every node against a model of +// the expected state, and cuts power, kills or stops nodes on the way. See README.md. +package main + +import ( + "bufio" + "encoding/json" + "errors" + "flag" + "fmt" + "log" + "os" + "path/filepath" + "time" +) + +func main() { + log.SetFlags(log.LstdFlags | log.Lmicroseconds) + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "usage: goclusterfs-soak run|replay|agent [flags]") + os.Exit(2) + } + var err error + switch os.Args[1] { + case "agent": + fs := flag.NewFlagSet("agent", flag.ExitOnError) + root := fs.String("root", mountPoint, "mount point to run operations under") + _ = fs.Parse(os.Args[2:]) + err = runAgent(*root, os.Stdin, os.Stdout) + case "run", "replay": + var cfg config + var oplog string + var upto uint64 + fs := flag.NewFlagSet(os.Args[1], flag.ExitOnError) + fs.StringVar(&cfg.checkout, "checkout", ".", "repository checkout with built goclusterfs-mount and goclusterfs-fsck") + fs.StringVar(&cfg.out, "out", filepath.Join("soak-out", time.Now().Format("20060102-150405")), "directory for the op log, cluster log and result") + fs.DurationVar(&cfg.duration, "duration", 48*time.Hour, "how long to run") + fs.IntVar(&cfg.lanes, "lanes", 3, "concurrent operation streams, each on its own files in the shared directories") + fs.Uint64Var(&cfg.seed, "seed", uint64(time.Now().UnixNano()), "random seed") + fs.DurationVar(&cfg.faultEvery, "fault-every", 10*time.Minute, "interval between faults; 0 for none") + fs.DurationVar(&cfg.fsckEvery, "fsck-every", time.Hour, "interval between quiesced fsck runs; 0 for none") + fs.DurationVar(&cfg.barrierEvery, "barrier-every", 5*time.Minute, "interval between full directory checks; 0 for none") + fs.DurationVar(&cfg.window, "window", 10*time.Second, "how long a node may take to show an acknowledged change") + fs.DurationVar(&cfg.stopFor, "stop-for", 25*time.Second, "how long a stop fault keeps the mount stopped; past the 10s heartbeat timeout") + fs.DurationVar(&cfg.opTimeout, "op-timeout", 5*time.Minute, "an operation taking longer is a hang") + fs.IntVar(&cfg.sshBasePort, "ssh-base-port", 3122, "host SSH port of VM 1; keep it apart from other VM runs") + fs.IntVar(&cfg.diskMB, "disk-mb", 4096, "shared disk size") + fs.Int64Var(&cfg.budget, "budget", 1200, "data blocks the model may hold, counting each snapshot as a full copy") + fs.StringVar(&oplog, "log", "", "replay: op log to run again") + fs.Uint64Var(&upto, "upto", 0, "replay: stop after this op sequence number") + _ = fs.Parse(os.Args[2:]) + if os.Args[1] == "run" { + err = run(cfg) + } else { + err = replay(cfg, oplog, upto) + } + default: + err = fmt.Errorf("unknown command %q", os.Args[1]) + } + if err != nil { + log.Fatal(err) + } +} + +// replay runs a logged soak again in log order, with the same operations, nodes, data and faults. +func replay(cfg config, oplog string, upto uint64) error { + if oplog == "" { + return errors.New("replay needs -log") + } + f, err := os.Open(oplog) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + var records []record + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 1<<20), 1<<20) + for sc.Scan() { + var r record + if err := json.Unmarshal(sc.Bytes(), &r); err != nil { + return fmt.Errorf("%s: %w", oplog, err) + } + records = append(records, r) + } + if err := sc.Err(); err != nil { + return err + } + + d, cleanup, err := newDriver(cfg) + if err != nil { + return err + } + defer cleanup() + if err := d.bringUp(); err != nil { + return fmt.Errorf("bring up the cluster: %w", err) + } + for _, r := range records { + if d.ctx.Err() != nil || upto > 0 && r.Type == "op" && r.Seq > upto { + break + } + switch r.Type { + case "op": + op := r.Op + logged := op.Want + op.Got, op.Err = 0, "" + d.model.mu.Lock() + op.Want = d.model.expect(&op) + d.model.mu.Unlock() + if op.Want != logged { + d.fail("op %d: the model expects errno %d where the run expected %d", op.Seq, op.Want, logged) + break + } + n := d.nodes[op.Node-1] + n.mu.RLock() + d.runOpLocked(&op, n) + case "fault": + d.injectFault(r.Fault, d.nodes[r.Victim-1]) + case "fsck": + d.quiescedFsck() + } + } + log.Print(d.report()) + return d.finish() +} diff --git a/tests/soak/model.go b/tests/soak/model.go new file mode 100644 index 00000000..009d31d4 --- /dev/null +++ b/tests/soak/model.go @@ -0,0 +1,428 @@ +//go:build linux + +package main + +import ( + "bytes" + "fmt" + "math/rand/v2" + "path" + "slices" + "strings" + "sync" + "syscall" +) + +const ( + blockSize = 1 << 20 + maxFile = 8 << 20 + // anyErrno is the expected result of an operation that must fail, with any errno. + anyErrno = -2 +) + +// sharedDirs hold files of every lane and are never removed. +var sharedDirs = []string{"/soak/d0", "/soak/d1", "/soak/d2", "/soak/d3"} + +// Op is one operation as the driver runs and logs it. +type Op struct { + Seq uint64 `json:"seq"` + Lane int `json:"lane"` + Node int `json:"node"` + Kind string `json:"op"` + Path string `json:"path,omitempty"` + Path2 string `json:"path2,omitempty"` + Name string `json:"name,omitempty"` + Off int64 `json:"off,omitempty"` + Len int64 `json:"len,omitempty"` + Seed uint64 `json:"seed,omitempty"` + Want int `json:"want"` + Got int `json:"got"` + Err string `json:"err,omitempty"` + At int64 `json:"at"` + Ms int64 `json:"ms"` +} + +func (op *Op) request() request { + r := request{Op: op.Kind, Path: op.Path, Path2: op.Path2, Off: op.Off, Len: op.Len, Seed: op.Seed} + switch op.Kind { + case "snapshot": + r.Op, r.Attr, r.Value = "setxattr", "user.snapshot", op.Name + case "clone": + r.Op, r.Attr, r.Value = "setxattr", "user.clone", op.Path2 + case "snapclone": + r.Op, r.Attr, r.Value = "setxattr", "user.snapshot.clone", op.Name+":"+op.Path2 + case "snapdelete": + r.Op, r.Attr, r.Value = "setxattr", "user.snapshot.delete", op.Name + } + return r +} + +// File is a file in the model. Its pointer is its identity: a later file at the same path is another File. +type File struct { + Data []byte + Snaps map[string]bool +} + +// Snap is a snapshot. Owner is nil once its file is gone, which makes it an orphan. +type Snap struct { + Data []byte + Owner *File +} + +// Model is the state every node must show once an acknowledged operation is visible. +type Model struct { + mu sync.Mutex + Files map[string]*File + Dirs map[string]bool + Snaps map[string]*Snap +} + +func newModel() *Model { + m := &Model{Files: map[string]*File{}, Dirs: map[string]bool{"/": true, "/soak": true}, Snaps: map[string]*Snap{}} + for _, d := range sharedDirs { + m.Dirs[d] = true + } + return m +} + +func (m *Model) exists(p string) bool { return m.Files[p] != nil || m.Dirs[p] } + +func (m *Model) hasChildren(dir string) bool { + for p := range m.Files { + if path.Dir(p) == dir { + return true + } + } + for p := range m.Dirs { + if p != dir && path.Dir(p) == dir { + return true + } + } + return false +} + +// expect returns the errno the operation must return on the model's current state. +func (m *Model) expect(op *Op) int { + f := m.Files[op.Path] + switch op.Kind { + case "create", "mkdir": + switch { + case m.exists(op.Path): + return int(syscall.EEXIST) + case !m.Dirs[path.Dir(op.Path)]: + return int(syscall.ENOENT) + } + case "write", "truncate", "unlink": + switch { + case m.Dirs[op.Path]: + return int(syscall.EISDIR) + case f == nil: + return int(syscall.ENOENT) + } + case "rmdir": + switch { + case f != nil: + return int(syscall.ENOTDIR) + case !m.Dirs[op.Path]: + return int(syscall.ENOENT) + case m.hasChildren(op.Path): + return int(syscall.ENOTEMPTY) + } + case "rename": + return anyErrno + case "snapshot": + switch { + case f == nil: + return int(syscall.ENOENT) + case m.Snaps[op.Name] != nil: + return int(syscall.EEXIST) + } + case "clone", "snapclone": + switch { + case f == nil: + return int(syscall.ENOENT) + case op.Kind == "snapclone" && (m.Snaps[op.Name] == nil || m.Snaps[op.Name].Owner != f): + return int(syscall.ENOENT) + case m.exists(op.Path2): + return int(syscall.EEXIST) + } + case "snapdelete": + if f == nil || m.Snaps[op.Name] == nil || m.Snaps[op.Name].Owner != f { + return int(syscall.ENOENT) + } + } + return 0 +} + +// apply changes the model as a successful operation changed the filesystem. +func (m *Model) apply(op *Op) { + f := m.Files[op.Path] + switch op.Kind { + case "create": + m.Files[op.Path] = &File{Snaps: map[string]bool{}} + case "write": + data := genData(op.Seed, op.Len) + if end := op.Off + op.Len; end > int64(len(f.Data)) { + f.Data = append(f.Data, make([]byte, end-int64(len(f.Data)))...) + } + copy(f.Data[op.Off:], data) + case "truncate": + if op.Len < int64(len(f.Data)) { + f.Data = f.Data[:op.Len] + } else { + f.Data = append(f.Data, make([]byte, op.Len-int64(len(f.Data)))...) + } + case "unlink": + for name := range f.Snaps { + m.Snaps[name].Owner = nil + } + delete(m.Files, op.Path) + case "mkdir": + m.Dirs[op.Path] = true + case "rmdir": + delete(m.Dirs, op.Path) + case "snapshot": + m.Snaps[op.Name] = &Snap{Data: bytes.Clone(f.Data), Owner: f} + f.Snaps[op.Name] = true + case "clone": + m.Files[op.Path2] = &File{Data: bytes.Clone(f.Data), Snaps: map[string]bool{}} + case "snapclone": + m.Files[op.Path2] = &File{Data: bytes.Clone(m.Snaps[op.Name].Data), Snaps: map[string]bool{}} + case "snapdelete": + delete(m.Snaps, op.Name) + delete(f.Snaps, op.Name) + } +} + +// blocks estimates the data blocks the model holds, counting each snapshot as a full copy. +func (m *Model) blocks() int64 { + var n int64 + for _, f := range m.Files { + n += (int64(len(f.Data)) + blockSize - 1) / blockSize + } + for _, s := range m.Snaps { + n += (int64(len(s.Data)) + blockSize - 1) / blockSize + } + return n +} + +// lane generates operations on its own files, directories and snapshots, whose names start with its prefix. +type lane struct { + id int + rng *rand.Rand + next int + budget int64 +} + +func (l *lane) prefix() string { return fmt.Sprintf("l%d-", l.id) } + +func (l *lane) name(kind string) string { + l.next++ + return fmt.Sprintf("%s%s%d", l.prefix(), kind, l.next) +} + +func (l *lane) owns(p string) bool { return strings.HasPrefix(path.Base(p), l.prefix()) } + +func pick[T any](r *rand.Rand, xs []T) T { return xs[r.IntN(len(xs))] } + +// generate picks the lane's next operation from the model's current state. The caller holds m.mu. +func (l *lane) generate(m *Model) *Op { + var files, dirs, snaps []string + orphans := 0 + for p := range m.Files { + if l.owns(p) { + files = append(files, p) + } + } + for p := range m.Dirs { + if l.owns(p) { + dirs = append(dirs, p) + } + } + for name, s := range m.Snaps { + switch { + case !l.owns(name): + case s.Owner != nil: + snaps = append(snaps, name) + default: + orphans++ + } + } + slices.Sort(files) + slices.Sort(dirs) + slices.Sort(snaps) + parents := append(slices.Clone(sharedDirs), dirs...) + full := m.blocks() > l.budget + r := l.rng + + weights := []struct { + kind string + w int + }{ + {"create", 15}, {"write", 30}, {"truncate", 8}, {"unlink", 8}, {"mkdir", 4}, {"rmdir", 4}, + {"rename", 2}, {"snapshot", 6}, {"clone", 3}, {"snapclone", 3}, {"snapdelete", 6}, {"invalid", 3}, + } + total := 0 + for _, w := range weights { + total += w.w + } + n := r.IntN(total) + kind := "" + for _, w := range weights { + if n -= w.w; n < 0 { + kind = w.kind + break + } + } + if full && slices.Contains([]string{"create", "write", "snapshot", "clone", "snapclone"}, kind) { + kind = pick(r, []string{"unlink", "truncate", "snapdelete"}) + } + if len(files) == 0 || len(files) < 6 && kind != "create" && kind != "mkdir" { + kind = "create" + } + if len(files) > 40 && kind == "create" { + kind = "unlink" + } + + op := &Op{Lane: l.id, Kind: kind} + switch kind { + case "create": + op.Path = path.Join(pick(r, parents), l.name("f")) + case "write": + op.Path = pick(r, files) + size := int64(len(m.Files[op.Path].Data)) + op.Len = pickLen(r) + switch v := r.IntN(10); { + case v < 5 && size > 0: + op.Off = r.Int64N(size) + case v < 8: + op.Off = size + default: + op.Off = size + r.Int64N(2*blockSize) + } + if op.Off+op.Len > maxFile { + op.Off, op.Len = 0, min(op.Len, 64<<10) + } + op.Seed = r.Uint64() + case "truncate": + op.Path = pick(r, files) + size := int64(len(m.Files[op.Path].Data)) + if r.IntN(2) == 0 || full { + op.Len = r.Int64N(size + 1) + } else { + op.Len = min(size+r.Int64N(blockSize), maxFile) + } + case "unlink": + op.Path = pick(r, files) + if f := m.Files[op.Path]; len(f.Snaps) > 0 && (orphans >= 20 || r.IntN(10) > 0) { + op.Kind, op.Name = "snapdelete", pickKey(r, f.Snaps) + } + case "mkdir": + if len(dirs) >= 6 { + op.Kind, op.Path = "rmdir", pick(r, dirs) + } else { + op.Path = path.Join(pick(r, sharedDirs), l.name("d")) + } + case "rmdir": + if len(dirs) == 0 { + op.Kind, op.Path = "mkdir", path.Join(pick(r, sharedDirs), l.name("d")) + break + } + op.Path = pick(r, dirs) + if m.hasChildren(op.Path) && r.IntN(2) == 0 { + for _, p := range files { + if path.Dir(p) == op.Path { + op.Kind, op.Path = "unlink", p + break + } + } + } + case "rename": + op.Path = pick(r, files) + op.Path2 = path.Join(path.Dir(op.Path), l.name("r")) + case "snapshot": + op.Path, op.Name = pick(r, files), l.name("s") + case "clone": + op.Path, op.Path2 = pick(r, files), "/"+l.name("c") + case "snapclone", "snapdelete": + if len(snaps) == 0 { + op.Kind, op.Path, op.Name = "snapshot", pick(r, files), l.name("s") + break + } + op.Name = pick(r, snaps) + op.Path = m.pathOf(m.Snaps[op.Name].Owner) + if kind == "snapclone" { + op.Path2 = "/" + l.name("c") + } + case "invalid": + switch r.IntN(3) { + case 0: + op.Kind, op.Path = "create", pick(r, files) + case 1: + op.Kind, op.Path, op.Off, op.Len, op.Seed = "write", path.Join(pick(r, parents), l.name("missing")), 0, 4096, r.Uint64() + default: + op.Kind, op.Path, op.Name, op.Path2 = "snapclone", pick(r, files), l.name("nosnap"), "/"+l.name("c") + } + } + return op +} + +func pickLen(r *rand.Rand) int64 { + switch v := r.IntN(20); { + case v < 14: + return 1 + r.Int64N(16<<10) + case v < 19: + return 64<<10 + r.Int64N(blockSize) + default: + return blockSize + r.Int64N(2*blockSize) + } +} + +func pickKey(r *rand.Rand, m map[string]bool) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.Sort(keys) + return pick(r, keys) +} + +func (m *Model) pathOf(f *File) string { + for p, g := range m.Files { + if g == f { + return p + } + } + return "" +} + +// snapsOf returns the names of the snapshots a node must list for the file at p. +func (m *Model) snapsOf(p string) []string { + f := m.Files[p] + if f == nil { + return nil + } + names := make([]string, 0, len(f.Snaps)) + for name := range f.Snaps { + names = append(names, name) + } + slices.Sort(names) + return names +} + +// listing returns what a node must list in dir, as the agent describes entries. +func (m *Model) listing(dir string) []string { + var out []string + for p, f := range m.Files { + if path.Dir(p) == dir { + out = append(out, fmt.Sprintf("f:%s:%d", path.Base(p), len(f.Data))) + } + } + for p := range m.Dirs { + if p != dir && path.Dir(p) == dir { + out = append(out, "d:"+path.Base(p)) + } + } + slices.Sort(out) + return out +} diff --git a/tests/soak/model_test.go b/tests/soak/model_test.go new file mode 100644 index 00000000..74f728fa --- /dev/null +++ b/tests/soak/model_test.go @@ -0,0 +1,70 @@ +//go:build linux + +package main + +import ( + "math/rand/v2" + "strings" + "syscall" + "testing" +) + +func step(t *testing.T, m *Model, op *Op, want int) { + t.Helper() + if got := m.expect(op); got != want { + t.Fatalf("%s %s: expect %d, want %d", op.Kind, op.Path, got, want) + } + if want == 0 { + m.apply(op) + } +} + +func TestModelFollowsTheFilesystemRules(t *testing.T) { + m := newModel() + f := "/soak/d0/l0-f1" + step(t, m, &Op{Kind: "create", Path: f}, 0) + step(t, m, &Op{Kind: "create", Path: f}, int(syscall.EEXIST)) + step(t, m, &Op{Kind: "write", Path: f, Off: 10, Len: 5, Seed: 7}, 0) + if data := m.Files[f].Data; len(data) != 15 || data[0] != 0 || string(data[10:]) != string(genData(7, 5)) { + t.Fatalf("write at 10 of an empty file left %v", data) + } + step(t, m, &Op{Kind: "snapshot", Path: f, Name: "l0-s1"}, 0) + step(t, m, &Op{Kind: "snapshot", Path: f, Name: "l0-s1"}, int(syscall.EEXIST)) + step(t, m, &Op{Kind: "mkdir", Path: "/soak/d0/l0-d1"}, 0) + step(t, m, &Op{Kind: "create", Path: "/soak/d0/l0-d1/l0-f2"}, 0) + step(t, m, &Op{Kind: "rmdir", Path: "/soak/d0/l0-d1"}, int(syscall.ENOTEMPTY)) + step(t, m, &Op{Kind: "unlink", Path: f}, 0) + if m.Snaps["l0-s1"].Owner != nil { + t.Fatal("the deleted file's snapshot still has an owner") + } + step(t, m, &Op{Kind: "create", Path: f}, 0) + step(t, m, &Op{Kind: "snapclone", Path: f, Name: "l0-s1", Path2: "/l0-c1"}, int(syscall.ENOENT)) + step(t, m, &Op{Kind: "snapdelete", Path: f, Name: "l0-s1"}, int(syscall.ENOENT)) + step(t, m, &Op{Kind: "rename", Path: f, Path2: "/soak/d0/l0-r1"}, anyErrno) +} + +// A lane only names and touches its own files, directories and snapshots, the same way for the same seed. +func TestLaneStaysInItsNamesAndRepeats(t *testing.T) { + gen := func() []string { + m := newModel() + l := &lane{id: 1, rng: rand.New(rand.NewPCG(42, 1)), budget: 1000} + var trace []string + for range 2000 { + op := l.generate(m) + for _, p := range []string{op.Path, op.Path2, op.Name} { + if p != "" && !strings.Contains(p, "/l1-") && !strings.HasPrefix(p, "l1-") { + t.Fatalf("%s names %q, outside lane 1", op.Kind, p) + } + } + if op.Kind != "rename" && m.expect(op) == 0 { + m.apply(op) + } + trace = append(trace, op.Kind+" "+op.Path) + } + return trace + } + a, b := gen(), gen() + if strings.Join(a, "\n") != strings.Join(b, "\n") { + t.Fatal("the same seed generated different operations") + } +} From cbee69149d16a7dd2c73836f47cefd4a7ab975cf Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 09:50:42 -0600 Subject: [PATCH 2/8] test(soak): find goclusterfs-mount by its command line, since process names are cut to 15 characters Co-Authored-By: Claude Opus 5.5 --- tests/soak/cluster.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/soak/cluster.sh b/tests/soak/cluster.sh index c9dfff7b..e7954861 100644 --- a/tests/soak/cluster.sh +++ b/tests/soak/cluster.sh @@ -23,13 +23,18 @@ source "${SOAK_DIR}/../vm/vm-test-harness.sh" >&2 reply() { echo "$*"; } +# Matches goclusterfs-mount by its command line: the kernel cuts process names to 15 +# characters, so pidof and pkill -x never match it. The bracket keeps the pattern from +# matching the shell that runs it. +MOUNT_PATTERN="'^/usr/local/bin/goclusterfs-moun[t]'" + mount_pid() { - ssh_run "$1" "pidof goclusterfs-mount" 2>/dev/null + ssh_run "$1" "pgrep -f $MOUNT_PATTERN" 2>/dev/null } # Kills a node's mount process and clears its mount point, as a crash leaves it. kill_mount() { - ssh_run "$1" "pkill -9 -x goclusterfs-mount; sleep 1; fusermount3 -uz ${GOCLUSTERFS_MOUNT} 2>/dev/null; true" + ssh_run "$1" "pkill -9 -f $MOUNT_PATTERN; sleep 1; fusermount3 -uz ${GOCLUSTERFS_MOUNT} 2>/dev/null; true" } remount() { @@ -39,7 +44,7 @@ remount() { unmount_node() { local vm_id=$1 waited=0 ssh_run "$vm_id" "fusermount3 -u ${GOCLUSTERFS_MOUNT}" || true - while ssh_run "$vm_id" "pidof goclusterfs-mount > /dev/null"; do + while ssh_run "$vm_id" "pgrep -f $MOUNT_PATTERN > /dev/null"; do if [[ $waited -ge 30 ]]; then log_warn "VM $vm_id: goclusterfs-mount still running 30s after unmount, killing it" kill_mount "$vm_id" From c1b777bf3e2ad034f4cc000554ef81013687e927 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 10:08:03 -0600 Subject: [PATCH 3/8] test(soak): expect ENOENT for a directory a lane removed during a full check Co-Authored-By: Claude Opus 5.5 --- tests/soak/driver.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/soak/driver.go b/tests/soak/driver.go index 9652cdd8..5cd7d43f 100644 --- a/tests/soak/driver.go +++ b/tests/soak/driver.go @@ -244,14 +244,21 @@ func (d *driver) compareSnaps(n *node, p string) error { return nil } +// compareDir checks node n lists dir as the model does. A lane may have removed dir since the caller listed it. func (d *driver) compareDir(n *node, dir string) error { d.model.mu.Lock() - want := d.model.listing(dir) + exists, want := d.model.Dirs[dir], d.model.listing(dir) d.model.mu.Unlock() resp, err := d.call(n, request{Op: "list", Path: dir}) if err != nil { return err } + if !exists { + if resp.Errno != int(syscall.ENOENT) { + return fmt.Errorf("%s: list errno %d, want ENOENT", dir, resp.Errno) + } + return nil + } got := slices.Clone(resp.Names) sort.Strings(got) if resp.Errno != 0 || !slices.Equal(got, want) { From de86afa82f7ba14574a66d4fab40cb85375585c8 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 10:17:00 -0600 Subject: [PATCH 4/8] test(soak): match goclusterfs-mount under gcompat's loader, whose command line starts with ld-linux Co-Authored-By: Claude Opus 5.5 --- tests/soak/cluster.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/soak/cluster.sh b/tests/soak/cluster.sh index e7954861..c60c44e7 100644 --- a/tests/soak/cluster.sh +++ b/tests/soak/cluster.sh @@ -24,9 +24,10 @@ source "${SOAK_DIR}/../vm/vm-test-harness.sh" >&2 reply() { echo "$*"; } # Matches goclusterfs-mount by its command line: the kernel cuts process names to 15 -# characters, so pidof and pkill -x never match it. The bracket keeps the pattern from -# matching the shell that runs it. -MOUNT_PATTERN="'^/usr/local/bin/goclusterfs-moun[t]'" +# characters, and in the VMs the binary runs under gcompat's loader, whose command line +# starts with ld-linux-x86-64.so.2. The bracket keeps the pattern from matching the shell +# that runs it. +MOUNT_PATTERN="'goclusterfs-moun[t] -mount'" mount_pid() { ssh_run "$1" "pgrep -f $MOUNT_PATTERN" 2>/dev/null From e58bb0041c1664b1a41a012b57bc9525baffb43c Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 11:21:17 -0600 Subject: [PATCH 5/8] test(soak): stop quietly when time is up, and log checks slower than the coherence window Operations cut short by the end of the run were reported as failures. The window grows to 30s, because reads on the surviving nodes wait out a failed node's detection, and every check slower than the documented 6s is logged as a slow record. Co-Authored-By: Claude Opus 5.5 --- tests/soak/README.md | 5 ++++- tests/soak/driver.go | 40 +++++++++++++++++++++++++--------------- tests/soak/main.go | 3 ++- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/tests/soak/README.md b/tests/soak/README.md index 84fc7071..70ac15eb 100644 --- a/tests/soak/README.md +++ b/tests/soak/README.md @@ -4,7 +4,10 @@ - **Operations:** create, write (overwrite, append, sparse), truncate, unlink, mkdir, rmdir, rename, snapshot, clone, clone from a snapshot, and snapshot delete. Rename is not supported, so each rename must fail and change nothing. - **Lanes:** several lanes run at once, each on its own files and snapshots, in directories they share. -- **Checks:** after every operation, each node must show its result, through a fresh open, within `-window` (10s). That covers `-attr-timeout` and `-cache-ttl`, because broadcast cannot cross the VMs' separate networks. +- **Checks:** after every operation, each node must show its result, through a fresh open, within `-window` (30s). + - The window covers `-attr-timeout` and `-cache-ttl`, because broadcast cannot cross the VMs' separate networks. + - It also covers waiting out a failed node's 10s detection. + - A check that needs longer than `-coherence` (6s) is logged as a `slow` record for review. - **Faults:** every `-fault-every`, one node is hit, picked at random from these: - power is cut (QEMU is killed); - goclusterfs-mount gets SIGKILL; diff --git a/tests/soak/driver.go b/tests/soak/driver.go index 5cd7d43f..6cf5661b 100644 --- a/tests/soak/driver.go +++ b/tests/soak/driver.go @@ -33,6 +33,7 @@ type config struct { fsckEvery time.Duration barrierEvery time.Duration window time.Duration + coherence time.Duration stopFor time.Duration opTimeout time.Duration sshBasePort int @@ -72,11 +73,13 @@ type driver struct { ctx context.Context cancel context.CancelFunc failMu sync.Mutex - failure string + // stopping is set when the run's time is up, so the operations it cuts short are not failures. + stopping atomic.Bool + failure string - ops, unexpected, faults, fscks, barriers atomic.Int64 - slowestMs atomic.Int64 - opsByKind sync.Map + ops, unexpected, faults, fscks, barriers, slow atomic.Int64 + slowestMs atomic.Int64 + opsByKind sync.Map } func (d *driver) logRecord(r record) { @@ -95,7 +98,7 @@ func (d *driver) logRecord(r record) { func (d *driver) fail(format string, args ...any) { d.failMu.Lock() defer d.failMu.Unlock() - if d.failure != "" { + if d.failure != "" || d.stopping.Load() { return } d.failure = fmt.Sprintf(format, args...) @@ -157,15 +160,21 @@ func (d *driver) upNodes() []*node { return up } -// converge retries check until it passes or the coherence window ends. The caller holds the node. -func (d *driver) converge(check func() error) error { +// converge retries check until it passes or the window ends. The caller holds the node. A check that passes only +// after the documented coherence window is logged for review: during a fault, reads can wait out the failure detection. +func (d *driver) converge(what string, check func() error) error { start := time.Now() for { err := check() if err == nil { - if ms := time.Since(start).Milliseconds(); ms > d.slowestMs.Load() { + took := time.Since(start) + if ms := took.Milliseconds(); ms > d.slowestMs.Load() { d.slowestMs.Store(ms) } + if took > d.cfg.coherence { + d.slow.Add(1) + d.logRecord(record{Type: "slow", Detail: fmt.Sprintf("%s took %s", what, took.Round(time.Millisecond))}) + } return nil } if errors.Is(err, errAgentGone) || time.Since(start) > d.cfg.window || d.ctx.Err() != nil { @@ -281,7 +290,7 @@ func (d *driver) verifyAll(n *node, content bool) error { sort.Strings(dirs) sort.Strings(files) for _, dir := range dirs { - if err := d.converge(func() error { return d.compareDir(n, dir) }); err != nil { + if err := d.converge(fmt.Sprintf("node %d dir %s", n.id, dir), func() error { return d.compareDir(n, dir) }); err != nil { return err } } @@ -289,10 +298,10 @@ func (d *driver) verifyAll(n *node, content bool) error { return nil } for _, p := range files { - if err := d.converge(func() error { return d.comparePath(n, p) }); err != nil { + if err := d.converge(fmt.Sprintf("node %d %s", n.id, p), func() error { return d.comparePath(n, p) }); err != nil { return err } - if err := d.converge(func() error { return d.compareSnaps(n, p) }); err != nil { + if err := d.converge(fmt.Sprintf("node %d snapshots of %s", n.id, p), func() error { return d.compareSnaps(n, p) }); err != nil { return err } } @@ -379,12 +388,12 @@ func (d *driver) check(op *Op) { func (d *driver) checkOp(m *node, op *Op, paths []string) error { for _, p := range paths { - if err := d.converge(func() error { return d.comparePath(m, p) }); err != nil { + if err := d.converge(fmt.Sprintf("node %d %s after op %d", m.id, p, op.Seq), func() error { return d.comparePath(m, p) }); err != nil { return err } } if slices.Contains([]string{"snapshot", "snapdelete", "snapclone"}, op.Kind) { - return d.converge(func() error { return d.compareSnaps(m, op.Path) }) + return d.converge(fmt.Sprintf("node %d snapshots of %s after op %d", m.id, op.Path, op.Seq), func() error { return d.compareSnaps(m, op.Path) }) } return nil } @@ -532,8 +541,8 @@ func (d *driver) report() string { return true }) sort.Strings(kinds) - return fmt.Sprintf("ops=%d unexpected=%d faults=%d fscks=%d barriers=%d slowest_converge_ms=%d files=%d snapshots=%d blocks=%d [%s]", - d.ops.Load(), d.unexpected.Load(), d.faults.Load(), d.fscks.Load(), d.barriers.Load(), d.slowestMs.Load(), + return fmt.Sprintf("ops=%d unexpected=%d faults=%d fscks=%d barriers=%d slow_checks=%d slowest_converge_ms=%d files=%d snapshots=%d blocks=%d [%s]", + d.ops.Load(), d.unexpected.Load(), d.faults.Load(), d.fscks.Load(), d.barriers.Load(), d.slow.Load(), d.slowestMs.Load(), files, snaps, blocks, strings.Join(kinds, " ")) } @@ -614,6 +623,7 @@ func run(cfg config) error { wg.Go(func() { every(ctx, cfg.barrierEvery, d.barrier) }) wg.Go(func() { every(ctx, time.Minute, func() { log.Print(d.report()) }) }) <-ctx.Done() + d.stopping.Store(true) d.cancel() wg.Wait() log.Print(d.report()) diff --git a/tests/soak/main.go b/tests/soak/main.go index e175d3b4..88761f3a 100644 --- a/tests/soak/main.go +++ b/tests/soak/main.go @@ -42,7 +42,8 @@ func main() { fs.DurationVar(&cfg.faultEvery, "fault-every", 10*time.Minute, "interval between faults; 0 for none") fs.DurationVar(&cfg.fsckEvery, "fsck-every", time.Hour, "interval between quiesced fsck runs; 0 for none") fs.DurationVar(&cfg.barrierEvery, "barrier-every", 5*time.Minute, "interval between full directory checks; 0 for none") - fs.DurationVar(&cfg.window, "window", 10*time.Second, "how long a node may take to show an acknowledged change") + fs.DurationVar(&cfg.window, "window", 30*time.Second, "how long a node may take to show an acknowledged change, including waiting out a failed node") + fs.DurationVar(&cfg.coherence, "coherence", 6*time.Second, "the documented coherence window; slower checks are logged") fs.DurationVar(&cfg.stopFor, "stop-for", 25*time.Second, "how long a stop fault keeps the mount stopped; past the 10s heartbeat timeout") fs.DurationVar(&cfg.opTimeout, "op-timeout", 5*time.Minute, "an operation taking longer is a hang") fs.IntVar(&cfg.sshBasePort, "ssh-base-port", 3122, "host SSH port of VM 1; keep it apart from other VM runs") From fda5aa2ce4b8dcd35841d4aac8b1866360b343c2 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 11:23:25 -0600 Subject: [PATCH 6/8] test: run the soak model's unit tests in CI and make test-unit Co-Authored-By: Claude Opus 5.5 --- Makefile | 2 +- tests/scripts/unit-groups.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 85b2981e..6e60aed6 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ DASHBOARD_DIST=internal/webui/build/dist GOLANGCI_LINT_VERSION=v2.15.0 GOVULNCHECK_VERSION=v1.9.0 COVER_PROFILE=coverage.out -UNIT_PKGS ?= ./internal/... +UNIT_PKGS ?= ./internal/... ./tests/soak UNIT_TIMEOUT ?= 20m # Default target diff --git a/tests/scripts/unit-groups.sh b/tests/scripts/unit-groups.sh index b3978bb9..0cf113ee 100755 --- a/tests/scripts/unit-groups.sh +++ b/tests/scripts/unit-groups.sh @@ -12,7 +12,7 @@ default_seconds=5 list_packages() { local out - out=$(go list ./internal/...) + out=$(go list ./internal/... ./tests/soak) echo "$out" } From d7e5497779b3b9849529764f9024046bd6d22f80 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 11:32:57 -0600 Subject: [PATCH 7/8] test(soak): name the errno match, which staticcheck flagged Co-Authored-By: Claude Opus 5.5 --- tests/soak/driver.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/soak/driver.go b/tests/soak/driver.go index 6cf5661b..6085b2ec 100644 --- a/tests/soak/driver.go +++ b/tests/soak/driver.go @@ -358,13 +358,21 @@ func (d *driver) finishOp(op *Op, resp response, err error) { d.check(op) } +// matches reports whether an operation that returned got did what the model expected. +func matches(want, got int) bool { + if want == anyErrno { + return got != 0 + } + return want == got +} + // check logs a finished operation, compares its errno with the model, applies it and checks every node shows it. func (d *driver) check(op *Op) { d.logRecord(record{Type: "op", Op: *op}) d.ops.Add(1) count, _ := d.opsByKind.LoadOrStore(op.Kind, new(atomic.Int64)) count.(*atomic.Int64).Add(1) - if !(op.Want == anyErrno && op.Got != 0 || op.Want == op.Got) { + if !matches(op.Want, op.Got) { d.unexpected.Add(1) d.fail("op %d %s %s on node %d: errno %d (%s), want %d", op.Seq, op.Kind, op.Path, op.Node, op.Got, op.Err, op.Want) return From 72edd316779def8b263960e0fb49cb19597b31ac Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Sat, 14 Nov 2026 11:45:48 -0600 Subject: [PATCH 8/8] test(soak): tell stale answers from slow calls in slow-check records Co-Authored-By: Claude Opus 5.5 --- tests/soak/driver.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/soak/driver.go b/tests/soak/driver.go index 6085b2ec..40ca37f8 100644 --- a/tests/soak/driver.go +++ b/tests/soak/driver.go @@ -164,8 +164,11 @@ func (d *driver) upNodes() []*node { // after the documented coherence window is logged for review: during a fault, reads can wait out the failure detection. func (d *driver) converge(what string, check func() error) error { start := time.Now() + stale, longestCall := 0, time.Duration(0) for { + callStart := time.Now() err := check() + longestCall = max(longestCall, time.Since(callStart)) if err == nil { took := time.Since(start) if ms := took.Milliseconds(); ms > d.slowestMs.Load() { @@ -173,10 +176,12 @@ func (d *driver) converge(what string, check func() error) error { } if took > d.cfg.coherence { d.slow.Add(1) - d.logRecord(record{Type: "slow", Detail: fmt.Sprintf("%s took %s", what, took.Round(time.Millisecond))}) + d.logRecord(record{Type: "slow", Detail: fmt.Sprintf("%s took %s: %d stale answers, longest call %s", + what, took.Round(time.Millisecond), stale, longestCall.Round(time.Millisecond))}) } return nil } + stale++ if errors.Is(err, errAgentGone) || time.Since(start) > d.cfg.window || d.ctx.Err() != nil { return fmt.Errorf("%v (still after %s)", err, time.Since(start).Round(time.Millisecond)) }