diff --git a/.gitignore b/.gitignore index cd338222..7605297a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ /goclusterfs /goclusterfs-mount /goclusterfs-fsck +/goclusterfs-soak +/soak-out/ /gfsctl /GoClusterFS /coverage.out diff --git a/Makefile b/Makefile index f987cd46..6e60aed6 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # GoClusterFS Makefile # Build and test automation for the distributed filesystem -.PHONY: build build-mount build-fsck build-gfsctl completions dashboard test test-unit test-integration test-all test-parity test-multinode-docker docker-build-test docker-clean-test test-multinode-vm vm-clean clean fmt vet check-fuse check-qemu lint vuln help +.PHONY: build build-mount build-fsck build-gfsctl soak completions dashboard test test-unit test-integration test-all test-parity test-multinode-docker docker-build-test docker-clean-test test-multinode-vm vm-clean clean fmt vet check-fuse check-qemu lint vuln help # Go parameters GOCMD=go @@ -22,7 +22,7 @@ DASHBOARD_DIST=internal/webui/build/dist GOLANGCI_LINT_VERSION=v2.15.0 GOVULNCHECK_VERSION=v1.9.0 COVER_PROFILE=coverage.out -UNIT_PKGS ?= ./internal/... +UNIT_PKGS ?= ./internal/... ./tests/soak UNIT_TIMEOUT ?= 20m # Default target @@ -41,6 +41,10 @@ build-fsck: $(GOBUILD) -o $(FSCK_BINARY) ./cmd/goclusterfs-fsck @echo "Built $(FSCK_BINARY) successfully" +# Static, so the same binary runs on the host as the driver and in the Alpine VMs as the agent +soak: build-mount build-fsck + CGO_ENABLED=0 $(GOBUILD) -o goclusterfs-soak ./tests/soak + build-gfsctl: @echo "Building $(GFSCTL_BINARY)..." $(GOBUILD) -o $(GFSCTL_BINARY) ./cmd/gfsctl diff --git a/tests/scripts/unit-groups.sh b/tests/scripts/unit-groups.sh index b3978bb9..0cf113ee 100755 --- a/tests/scripts/unit-groups.sh +++ b/tests/scripts/unit-groups.sh @@ -12,7 +12,7 @@ default_seconds=5 list_packages() { local out - out=$(go list ./internal/...) + out=$(go list ./internal/... ./tests/soak) echo "$out" } diff --git a/tests/soak/README.md b/tests/soak/README.md new file mode 100644 index 00000000..70ac15eb --- /dev/null +++ b/tests/soak/README.md @@ -0,0 +1,39 @@ +# Multi-node soak + +`goclusterfs-soak` runs a random mix of operations on a 3-VM cluster for hours or days and checks every node against a model of the expected state. + +- **Operations:** create, write (overwrite, append, sparse), truncate, unlink, mkdir, rmdir, rename, snapshot, clone, clone from a snapshot, and snapshot delete. Rename is not supported, so each rename must fail and change nothing. +- **Lanes:** several lanes run at once, each on its own files and snapshots, in directories they share. +- **Checks:** after every operation, each node must show its result, through a fresh open, within `-window` (30s). + - The window covers `-attr-timeout` and `-cache-ttl`, because broadcast cannot cross the VMs' separate networks. + - It also covers waiting out a failed node's 10s detection. + - A check that needs longer than `-coherence` (6s) is logged as a `slow` record for review. +- **Faults:** every `-fault-every`, one node is hit, picked at random from these: + - power is cut (QEMU is killed); + - goclusterfs-mount gets SIGKILL; + - goclusterfs-mount is stopped past the 10s heartbeat timeout and resumed. + + The node comes back, mounts again and must show the model. The other nodes keep working through the fault. +- **fsck:** every `-fsck-every`, all operations stop, every node unmounts and goclusterfs-fsck runs. Anything other than leaks and orphan snapshots is a failure. Leaks are repaired. + +The first failure stops the run. `result.txt` says what failed, `ops.jsonl` logs every operation, fault and fsck, and `cluster.img` keeps the shared disk. + +## Run + +On a host with KVM, from a checkout of its own: + +```bash +make soak +flock /tmp/goclusterfs-soak.lock systemd-run --scope -p MemoryMax=8G -p CPUQuota=400% --quiet \ + ./goclusterfs-soak run -duration 48h -out /root/gcfs-soak/run1 +``` + +The VMs use SSH ports from `-ssh-base-port` (3122), so the soak can share a host with other VM test runs. It logs a summary every minute. + +## Replay + +```bash +./goclusterfs-soak replay -log /root/gcfs-soak/run1/ops.jsonl -out /root/gcfs-soak/replay1 +``` + +Replay boots a new cluster and runs the logged operations one at a time, in log order, on the same nodes, with the same data, faults and fsck runs. `-upto N` stops after operation N. Lanes ran at the same time in the original run, so a failure that needs their exact timing may not happen again. diff --git a/tests/soak/agent.go b/tests/soak/agent.go new file mode 100644 index 00000000..49acc71f --- /dev/null +++ b/tests/soak/agent.go @@ -0,0 +1,174 @@ +//go:build linux + +package main + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand/v2" + "os" + "path/filepath" + "sync" + "syscall" + + "golang.org/x/sys/unix" +) + +// request is one operation the driver asks an agent to run. Paths are relative to the mount. +type request struct { + ID uint64 `json:"id"` + Op string `json:"op"` + Path string `json:"path,omitempty"` + Path2 string `json:"path2,omitempty"` + Attr string `json:"attr,omitempty"` + Value string `json:"value,omitempty"` + Off int64 `json:"off,omitempty"` + Len int64 `json:"len,omitempty"` + Seed uint64 `json:"seed,omitempty"` +} + +type response struct { + ID uint64 `json:"id"` + Errno int `json:"errno"` + Err string `json:"err,omitempty"` + Size int64 `json:"size,omitempty"` + Sum string `json:"sum,omitempty"` + Names []string `json:"names,omitempty"` + Text string `json:"text,omitempty"` +} + +// genData returns n bytes that depend only on seed, so the driver and the agents produce the same data. +func genData(seed uint64, n int64) []byte { + b := make([]byte, n) + r := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15)) + for i := int64(0); i < n; i += 8 { + v := r.Uint64() + for j := int64(0); j < 8 && i+j < n; j++ { + b[i+j] = byte(v >> (8 * j)) + } + } + return b +} + +func sum(b []byte) string { + h := sha256.Sum256(b) + return hex.EncodeToString(h[:8]) +} + +// runAgent serves requests from stdin until it closes, running each in its own goroutine. +func runAgent(root string, in io.Reader, out io.Writer) error { + dec := json.NewDecoder(in) + enc := json.NewEncoder(out) + var mu sync.Mutex + var wg sync.WaitGroup + defer wg.Wait() + for { + var req request + if err := dec.Decode(&req); err != nil { + if errors.Is(err, io.EOF) { + return nil + } + return err + } + wg.Add(1) + go func() { + defer wg.Done() + resp := serve(root, req) + resp.ID = req.ID + mu.Lock() + defer mu.Unlock() + if err := enc.Encode(resp); err != nil { + fmt.Fprintf(os.Stderr, "agent: reply %d: %v\n", req.ID, err) + } + }() + } +} + +func serve(root string, req request) response { + var resp response + p := filepath.Join(root, req.Path) + var err error + switch req.Op { + case "ping": + case "create": + var f *os.File + if f, err = os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644); err == nil { + err = f.Close() + } + case "write": + err = writeSynced(p, req.Off, genData(req.Seed, req.Len)) + case "truncate": + err = unix.Truncate(p, req.Len) + case "unlink": + err = unix.Unlink(p) + case "mkdir": + err = unix.Mkdir(p, 0o755) + case "rmdir": + err = unix.Rmdir(p) + case "rename": + err = unix.Rename(p, filepath.Join(root, req.Path2)) + case "setxattr": + err = unix.Setxattr(p, req.Attr, []byte(req.Value), 0) + case "getxattr": + buf := make([]byte, 64<<10) + var n int + if n, err = unix.Getxattr(p, req.Attr, buf); err == nil { + resp.Text = string(buf[:n]) + } + case "read": + var data []byte + if data, err = os.ReadFile(p); err == nil { + resp.Size, resp.Sum = int64(len(data)), sum(data) + } + case "list": + var entries []os.DirEntry + if entries, err = os.ReadDir(p); err == nil { + for _, e := range entries { + resp.Names = append(resp.Names, entryName(p, e)) + } + } + default: + err = fmt.Errorf("unknown op %q", req.Op) + } + if err != nil { + resp.Err = err.Error() + resp.Errno = -1 + var errno syscall.Errno + if errors.As(err, &errno) { + resp.Errno = int(errno) + } + } + return resp +} + +// entryName describes a directory entry as "d:name" or "f:name:size". +func entryName(dir string, e os.DirEntry) string { + if e.IsDir() { + return "d:" + e.Name() + } + var st unix.Stat_t + if err := unix.Lstat(filepath.Join(dir, e.Name()), &st); err != nil { + return "f:" + e.Name() + ":?" + } + return fmt.Sprintf("f:%s:%d", e.Name(), st.Size) +} + +func writeSynced(p string, off int64, data []byte) error { + f, err := os.OpenFile(p, os.O_WRONLY, 0) + if err != nil { + return err + } + if _, err := f.WriteAt(data, off); err != nil { + _ = f.Close() + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + return f.Close() +} diff --git a/tests/soak/cluster.go b/tests/soak/cluster.go new file mode 100644 index 00000000..afa4b9c7 --- /dev/null +++ b/tests/soak/cluster.go @@ -0,0 +1,189 @@ +//go:build linux + +package main + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + "sync" + "sync/atomic" +) + +// controller drives tests/soak/cluster.sh, which owns the VMs. +type controller struct { + mu sync.Mutex + cmd *exec.Cmd + in io.WriteCloser + out *bufio.Scanner +} + +func startController(script string, env []string, logw io.Writer) (*controller, error) { + cmd := exec.Command("bash", script) + cmd.Env = append(os.Environ(), env...) + cmd.Stderr = logw + in, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + out, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + sc := bufio.NewScanner(out) + sc.Buffer(make([]byte, 1<<20), 1<<20) + return &controller{cmd: cmd, in: in, out: sc}, nil +} + +// run sends one command and returns the text after "ok" and any "out" lines before it. +func (c *controller) run(command string) (string, []string, error) { + c.mu.Lock() + defer c.mu.Unlock() + if _, err := fmt.Fprintln(c.in, command); err != nil { + return "", nil, err + } + var lines []string + for c.out.Scan() { + line := c.out.Text() + switch { + case strings.HasPrefix(line, "out "): + lines = append(lines, strings.TrimPrefix(line, "out ")) + case line == "ok" || strings.HasPrefix(line, "ok "): + return strings.TrimSpace(strings.TrimPrefix(line, "ok")), lines, nil + case strings.HasPrefix(line, "err"): + return "", lines, fmt.Errorf("controller: %s", line) + } + } + return "", lines, fmt.Errorf("controller exited during %q: %v", command, c.out.Err()) +} + +func (c *controller) stop() { + _, _, _ = c.run("down") + _ = c.in.Close() + _ = c.cmd.Wait() +} + +func sshArgs(key string) []string { + return []string{ + "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR", + "-o", "BatchMode=yes", "-o", "IdentitiesOnly=yes", "-o", "ServerAliveInterval=5", "-o", "ServerAliveCountMax=6", + "-i", key, + } +} + +// pushAgent copies this binary to a VM, whose root file system does not survive a reboot. +func pushAgent(key string, port int) error { + exe, err := os.Executable() + if err != nil { + return err + } + args := append(sshArgs(key), "-P", fmt.Sprint(port), exe, "root@127.0.0.1:/usr/local/bin/goclusterfs-soak") + if out, err := exec.Command("scp", args...).CombinedOutput(); err != nil { + return fmt.Errorf("scp to port %d: %v: %s", port, err, out) + } + return nil +} + +// agentConn is an ssh session to the agent on one VM. Calls may overlap. +type agentConn struct { + mu sync.Mutex + enc *json.Encoder + pending map[uint64]chan response + nextID atomic.Uint64 + cmd *exec.Cmd + stdin io.WriteCloser + done chan struct{} + err error +} + +var errAgentGone = errors.New("agent connection closed") + +func dialAgent(key string, port int, mount string, logw io.Writer) (*agentConn, error) { + args := append(sshArgs(key), "-p", fmt.Sprint(port), "root@127.0.0.1", + "/usr/local/bin/goclusterfs-soak", "agent", "-root", mount) + cmd := exec.Command("ssh", args...) + cmd.Stderr = logw + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + a := &agentConn{enc: json.NewEncoder(stdin), pending: map[uint64]chan response{}, cmd: cmd, stdin: stdin, done: make(chan struct{})} + go a.read(stdout) + if _, err := a.call(context.Background(), request{Op: "ping"}); err != nil { + a.close() + return nil, fmt.Errorf("agent on port %d: %w", port, err) + } + return a, nil +} + +func (a *agentConn) read(r io.Reader) { + dec := json.NewDecoder(r) + for { + var resp response + if err := dec.Decode(&resp); err != nil { + a.mu.Lock() + a.err = fmt.Errorf("%w: %v", errAgentGone, err) + for id, ch := range a.pending { + close(ch) + delete(a.pending, id) + } + a.mu.Unlock() + close(a.done) + return + } + a.mu.Lock() + ch := a.pending[resp.ID] + delete(a.pending, resp.ID) + a.mu.Unlock() + if ch != nil { + ch <- resp + } + } +} + +func (a *agentConn) call(ctx context.Context, req request) (response, error) { + req.ID = a.nextID.Add(1) + ch := make(chan response, 1) + a.mu.Lock() + if a.err != nil { + a.mu.Unlock() + return response{}, a.err + } + a.pending[req.ID] = ch + err := a.enc.Encode(req) + a.mu.Unlock() + if err != nil { + return response{}, fmt.Errorf("%w: %v", errAgentGone, err) + } + select { + case resp, ok := <-ch: + if !ok { + return response{}, a.err + } + return resp, nil + case <-ctx.Done(): + return response{}, ctx.Err() + } +} + +func (a *agentConn) close() { + _ = a.stdin.Close() + _ = a.cmd.Process.Kill() + _ = a.cmd.Wait() +} diff --git a/tests/soak/cluster.sh b/tests/soak/cluster.sh new file mode 100644 index 00000000..c60c44e7 --- /dev/null +++ b/tests/soak/cluster.sh @@ -0,0 +1,135 @@ +#!/bin/bash +# Cluster controller for the soak driver. It owns the VMs, which die with it, and +# answers one command per stdin line with one "ok ..." or "err ..." line on stdout. +# Everything else goes to stderr. +# +# up boot NODE_COUNT VMs, format and mount; answers "ok ..." +# power VM cut power to VM, boot it again and mount +# sigkill VM SIGKILL VM's goclusterfs-mount and mount again +# stop VM SECONDS SIGSTOP VM's goclusterfs-mount for SECONDS, resume it, then mount again +# remount VM kill VM's goclusterfs-mount and mount again +# unmount-all unmount every node +# mount-all mount every node that is not mounted +# fsck [-repair] run goclusterfs-fsck on VM 1 with every node unmounted; answers +# "out " for each problem line, then "ok rc=" +# save-image DEST copy the shared disk to DEST +# down stop every VM and exit + +set -uo pipefail + +SOAK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +NODE_COUNT=${NODE_COUNT:-3} +source "${SOAK_DIR}/../vm/vm-test-harness.sh" >&2 + +reply() { echo "$*"; } + +# Matches goclusterfs-mount by its command line: the kernel cuts process names to 15 +# characters, and in the VMs the binary runs under gcompat's loader, whose command line +# starts with ld-linux-x86-64.so.2. The bracket keeps the pattern from matching the shell +# that runs it. +MOUNT_PATTERN="'goclusterfs-moun[t] -mount'" + +mount_pid() { + ssh_run "$1" "pgrep -f $MOUNT_PATTERN" 2>/dev/null +} + +# Kills a node's mount process and clears its mount point, as a crash leaves it. +kill_mount() { + ssh_run "$1" "pkill -9 -f $MOUNT_PATTERN; sleep 1; fusermount3 -uz ${GOCLUSTERFS_MOUNT} 2>/dev/null; true" +} + +remount() { + kill_mount "$1" && setup_goclusterfs "$1" false +} + +unmount_node() { + local vm_id=$1 waited=0 + ssh_run "$vm_id" "fusermount3 -u ${GOCLUSTERFS_MOUNT}" || true + while ssh_run "$vm_id" "pgrep -f $MOUNT_PATTERN > /dev/null"; do + if [[ $waited -ge 30 ]]; then + log_warn "VM $vm_id: goclusterfs-mount still running 30s after unmount, killing it" + kill_mount "$vm_id" + return 0 + fi + sleep 1 + waited=$((waited + 1)) + done +} + +handle() { + local cmd=$1 + shift + case "$cmd" in + up) + create_shared_disk >&2 || return 1 + local vm_id ports=() + for vm_id in $(seq 1 "$NODE_COUNT"); do + start_vm "$vm_id" "$([[ $vm_id -eq 1 ]] && echo true || echo false)" >&2 || return 1 + done + for vm_id in $(seq 1 "$NODE_COUNT"); do + wait_vm_healthy "$vm_id" >&2 || return 1 + setup_goclusterfs "$vm_id" "$([[ $vm_id -eq 1 ]] && echo true || echo false)" >&2 || return 1 + ports+=("$(get_vm_ssh_port "$vm_id")") + done + reply "ok $(get_ssh_key) ${ports[*]}" + ;; + power) + kill_vm "$1" >&2 && start_vm "$1" false >&2 && wait_vm_healthy "$1" >&2 && + setup_goclusterfs "$1" false >&2 && reply ok + ;; + sigkill) + remount "$1" >&2 && reply ok + ;; + stop) + local pid + pid=$(mount_pid "$1") || return 1 + ssh_run "$1" "kill -STOP $pid" >&2 || return 1 + sleep "$2" + ssh_run "$1" "kill -CONT $pid" >&2 + # The other nodes declared it failed and fenced it, so it has to mount again. + remount "$1" >&2 && reply ok + ;; + remount) + remount "$1" >&2 && reply ok + ;; + unmount-all) + local vm_id + for vm_id in $(seq 1 "$NODE_COUNT"); do + unmount_node "$vm_id" >&2 + done + reply ok + ;; + mount-all) + local vm_id + for vm_id in $(seq 1 "$NODE_COUNT"); do + if ! ssh_run "$vm_id" "mountpoint -q ${GOCLUSTERFS_MOUNT}"; then + setup_goclusterfs "$vm_id" false >&2 || return 1 + fi + done + reply ok + ;; + fsck) + local out rc + out=$(ssh_run 1 "/usr/local/bin/goclusterfs-fsck ${1:-} /dev/vda 2>&1; echo rc=\$?") + rc=${out##*rc=} + echo "$out" | grep -E '^ *\[(WARN|ERROR)\]' | sed 's/^ */out /' + reply "ok rc=$rc" + ;; + save-image) + cp --sparse=always "${SHARED_DISK_PATH}" "$1" >&2 && reply ok + ;; + down) + reply ok + exit 0 + ;; + *) + reply "err unknown command $cmd" + return 0 + ;; + esac +} + +while read -r line; do + # shellcheck disable=SC2086 + handle $line || reply "err $line failed" +done diff --git a/tests/soak/driver.go b/tests/soak/driver.go new file mode 100644 index 00000000..40ca37f8 --- /dev/null +++ b/tests/soak/driver.go @@ -0,0 +1,659 @@ +//go:build linux + +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "math/rand/v2" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" +) + +const mountPoint = "/mnt/goclusterfs" + +type config struct { + checkout string + out string + duration time.Duration + lanes int + seed uint64 + faultEvery time.Duration + fsckEvery time.Duration + barrierEvery time.Duration + window time.Duration + coherence time.Duration + stopFor time.Duration + opTimeout time.Duration + sshBasePort int + diskMB int + budget int64 +} + +type node struct { + id int + port int + mu sync.RWMutex // an operation holds it shared; a fault holds it exclusively + up atomic.Bool + agent *agentConn +} + +// record is one line of the operation log. +type record struct { + Type string `json:"type"` + Op + Fault string `json:"fault,omitempty"` + Victim int `json:"victim,omitempty"` + Detail string `json:"detail,omitempty"` +} + +type driver struct { + cfg config + ctl *controller + key string + nodes []*node + model *Model + pause sync.RWMutex // lanes hold it shared per operation; barriers and fsck hold it exclusively + faultMu sync.Mutex + seq atomic.Uint64 + logMu sync.Mutex + logw io.Writer + stderr io.Writer + ctx context.Context + cancel context.CancelFunc + failMu sync.Mutex + // stopping is set when the run's time is up, so the operations it cuts short are not failures. + stopping atomic.Bool + failure string + + ops, unexpected, faults, fscks, barriers, slow atomic.Int64 + slowestMs atomic.Int64 + opsByKind sync.Map +} + +func (d *driver) logRecord(r record) { + d.logMu.Lock() + defer d.logMu.Unlock() + b, err := json.Marshal(r) + if err == nil { + _, err = d.logw.Write(append(b, '\n')) + } + if err != nil { + log.Printf("op log: %v", err) + } +} + +// fail stops the soak at the first failure and keeps its description. +func (d *driver) fail(format string, args ...any) { + d.failMu.Lock() + defer d.failMu.Unlock() + if d.failure != "" || d.stopping.Load() { + return + } + d.failure = fmt.Sprintf(format, args...) + log.Printf("FAILURE: %s", d.failure) + d.logRecord(record{Type: "failure", Detail: d.failure}) + d.cancel() +} + +func (d *driver) bringUp() error { + reply, _, err := d.ctl.run("up") + if err != nil { + return err + } + fields := strings.Fields(reply) + if len(fields) < 2 { + return fmt.Errorf("controller answered up with %q", reply) + } + d.key = fields[0] + for i, p := range fields[1:] { + n := &node{id: i + 1} + if _, err := fmt.Sscan(p, &n.port); err != nil { + return err + } + d.nodes = append(d.nodes, n) + } + for _, n := range d.nodes { + if err := d.connect(n); err != nil { + return err + } + } + for _, dir := range append([]string{"/soak"}, sharedDirs...) { + if resp, err := d.nodes[0].agent.call(d.ctx, request{Op: "mkdir", Path: dir}); err != nil || resp.Errno != 0 { + return fmt.Errorf("mkdir %s: %v %s", dir, err, resp.Err) + } + } + return nil +} + +func (d *driver) connect(n *node) error { + if err := pushAgent(d.key, n.port); err != nil { + return err + } + a, err := dialAgent(d.key, n.port, mountPoint, d.stderr) + if err != nil { + return err + } + n.agent = a + n.up.Store(true) + return nil +} + +func (d *driver) upNodes() []*node { + var up []*node + for _, n := range d.nodes { + if n.up.Load() { + up = append(up, n) + } + } + return up +} + +// converge retries check until it passes or the window ends. The caller holds the node. A check that passes only +// after the documented coherence window is logged for review: during a fault, reads can wait out the failure detection. +func (d *driver) converge(what string, check func() error) error { + start := time.Now() + stale, longestCall := 0, time.Duration(0) + for { + callStart := time.Now() + err := check() + longestCall = max(longestCall, time.Since(callStart)) + if err == nil { + took := time.Since(start) + if ms := took.Milliseconds(); ms > d.slowestMs.Load() { + d.slowestMs.Store(ms) + } + if took > d.cfg.coherence { + d.slow.Add(1) + d.logRecord(record{Type: "slow", Detail: fmt.Sprintf("%s took %s: %d stale answers, longest call %s", + what, took.Round(time.Millisecond), stale, longestCall.Round(time.Millisecond))}) + } + return nil + } + stale++ + if errors.Is(err, errAgentGone) || time.Since(start) > d.cfg.window || d.ctx.Err() != nil { + return fmt.Errorf("%v (still after %s)", err, time.Since(start).Round(time.Millisecond)) + } + time.Sleep(100 * time.Millisecond) + } +} + +func (d *driver) call(n *node, req request) (response, error) { + ctx, cancel := context.WithTimeout(d.ctx, d.cfg.opTimeout) + defer cancel() + return n.agent.call(ctx, req) +} + +// comparePath checks what node n shows at p against the model. +func (d *driver) comparePath(n *node, p string) error { + d.model.mu.Lock() + f, isDir := d.model.Files[p], d.model.Dirs[p] + var size int64 + var want string + if f != nil { + size, want = int64(len(f.Data)), sum(f.Data) + } + d.model.mu.Unlock() + switch { + case f != nil: + resp, err := d.call(n, request{Op: "read", Path: p}) + if err != nil { + return err + } + if resp.Errno != 0 || resp.Size != size || resp.Sum != want { + return fmt.Errorf("%s: read errno %d size %d sum %s, want size %d sum %s", p, resp.Errno, resp.Size, resp.Sum, size, want) + } + case isDir: + resp, err := d.call(n, request{Op: "list", Path: p}) + if err != nil { + return err + } + if resp.Errno != 0 { + return fmt.Errorf("%s: list errno %d (%s), want a directory", p, resp.Errno, resp.Err) + } + default: + resp, err := d.call(n, request{Op: "read", Path: p}) + if err != nil { + return err + } + if resp.Errno != int(syscall.ENOENT) { + return fmt.Errorf("%s: read errno %d size %d, want ENOENT", p, resp.Errno, resp.Size) + } + } + return nil +} + +func (d *driver) compareSnaps(n *node, p string) error { + d.model.mu.Lock() + want, exists := d.model.snapsOf(p), d.model.Files[p] != nil + d.model.mu.Unlock() + if !exists { + return nil + } + resp, err := d.call(n, request{Op: "getxattr", Path: p, Attr: "user.snapshots"}) + if err != nil { + return err + } + var got []string + for _, line := range strings.FieldsFunc(resp.Text, func(r rune) bool { return r == '\n' || r == ',' }) { + if name, state, ok := strings.Cut(line, ":"); ok && state == "active" { + got = append(got, name) + } + } + sort.Strings(got) + if resp.Errno != 0 || !slices.Equal(got, want) { + return fmt.Errorf("%s: snapshots %v (errno %d), want %v", p, got, resp.Errno, want) + } + return nil +} + +// compareDir checks node n lists dir as the model does. A lane may have removed dir since the caller listed it. +func (d *driver) compareDir(n *node, dir string) error { + d.model.mu.Lock() + exists, want := d.model.Dirs[dir], d.model.listing(dir) + d.model.mu.Unlock() + resp, err := d.call(n, request{Op: "list", Path: dir}) + if err != nil { + return err + } + if !exists { + if resp.Errno != int(syscall.ENOENT) { + return fmt.Errorf("%s: list errno %d, want ENOENT", dir, resp.Errno) + } + return nil + } + got := slices.Clone(resp.Names) + sort.Strings(got) + if resp.Errno != 0 || !slices.Equal(got, want) { + return fmt.Errorf("%s: lists %v (errno %d), want %v", dir, got, resp.Errno, want) + } + return nil +} + +// verifyAll checks every directory listing, and with content every file and its snapshots, on node n. +func (d *driver) verifyAll(n *node, content bool) error { + d.model.mu.Lock() + var dirs, files []string + for p := range d.model.Dirs { + dirs = append(dirs, p) + } + for p := range d.model.Files { + files = append(files, p) + } + d.model.mu.Unlock() + sort.Strings(dirs) + sort.Strings(files) + for _, dir := range dirs { + if err := d.converge(fmt.Sprintf("node %d dir %s", n.id, dir), func() error { return d.compareDir(n, dir) }); err != nil { + return err + } + } + if !content { + return nil + } + for _, p := range files { + if err := d.converge(fmt.Sprintf("node %d %s", n.id, p), func() error { return d.comparePath(n, p) }); err != nil { + return err + } + if err := d.converge(fmt.Sprintf("node %d snapshots of %s", n.id, p), func() error { return d.compareSnaps(n, p) }); err != nil { + return err + } + } + return nil +} + +func (d *driver) runLane(l *lane) { + for d.ctx.Err() == nil { + d.pause.RLock() + d.model.mu.Lock() + op := l.generate(d.model) + op.Seq = d.seq.Add(1) + op.Want = d.model.expect(op) + d.model.mu.Unlock() + for d.ctx.Err() == nil { + up := d.upNodes() + if len(up) == 0 { + time.Sleep(100 * time.Millisecond) + continue + } + n := pick(l.rng, up) + n.mu.RLock() + if !n.up.Load() { + n.mu.RUnlock() + continue + } + d.runOpLocked(op, n) + break + } + d.pause.RUnlock() + } +} + +// runOpLocked runs op on n, which the caller holds shared, and releases n before checking the other nodes. +func (d *driver) runOpLocked(op *Op, n *node) { + op.Node = n.id + op.At = time.Now().UnixMilli() + ctx, cancel := context.WithTimeout(d.ctx, d.cfg.opTimeout) + resp, err := n.agent.call(ctx, op.request()) + cancel() + n.mu.RUnlock() + d.finishOp(op, resp, err) +} + +func (d *driver) finishOp(op *Op, resp response, err error) { + op.Ms = time.Now().UnixMilli() - op.At + if err != nil { + if d.ctx.Err() == nil { + d.fail("op %d %s %s on node %d: %v", op.Seq, op.Kind, op.Path, op.Node, err) + } + return + } + op.Got, op.Err = resp.Errno, resp.Err + d.check(op) +} + +// matches reports whether an operation that returned got did what the model expected. +func matches(want, got int) bool { + if want == anyErrno { + return got != 0 + } + return want == got +} + +// check logs a finished operation, compares its errno with the model, applies it and checks every node shows it. +func (d *driver) check(op *Op) { + d.logRecord(record{Type: "op", Op: *op}) + d.ops.Add(1) + count, _ := d.opsByKind.LoadOrStore(op.Kind, new(atomic.Int64)) + count.(*atomic.Int64).Add(1) + if !matches(op.Want, op.Got) { + d.unexpected.Add(1) + d.fail("op %d %s %s on node %d: errno %d (%s), want %d", op.Seq, op.Kind, op.Path, op.Node, op.Got, op.Err, op.Want) + return + } + if op.Got == 0 { + d.model.mu.Lock() + d.model.apply(op) + d.model.mu.Unlock() + } + paths := []string{op.Path} + if op.Path2 != "" { + paths = append(paths, op.Path2) + } + for _, m := range d.upNodes() { + if err := d.onNode(m, func() error { return d.checkOp(m, op, paths) }); err != nil { + d.fail("after op %d %s %s on node %d: node %d: %v", op.Seq, op.Kind, op.Path, op.Node, m.id, err) + return + } + } +} + +func (d *driver) checkOp(m *node, op *Op, paths []string) error { + for _, p := range paths { + if err := d.converge(fmt.Sprintf("node %d %s after op %d", m.id, p, op.Seq), func() error { return d.comparePath(m, p) }); err != nil { + return err + } + } + if slices.Contains([]string{"snapshot", "snapdelete", "snapclone"}, op.Kind) { + return d.converge(fmt.Sprintf("node %d snapshots of %s after op %d", m.id, op.Path, op.Seq), func() error { return d.compareSnaps(m, op.Path) }) + } + return nil +} + +// onNode runs f while holding node n shared, unless a fault has taken n down. +func (d *driver) onNode(n *node, f func() error) error { + if !n.up.Load() { + return nil + } + n.mu.RLock() + defer n.mu.RUnlock() + if !n.up.Load() { + return nil + } + return f() +} + +// injectFault takes one node down, brings it back and checks it shows the model. Operations go on elsewhere. +func (d *driver) injectFault(kind string, n *node) { + d.faultMu.Lock() + defer d.faultMu.Unlock() + start := time.Now() + n.up.Store(false) + n.mu.Lock() + defer n.mu.Unlock() + n.agent.close() + command := fmt.Sprintf("%s %d", kind, n.id) + if kind == "stop" { + command = fmt.Sprintf("stop %d %d", n.id, int(d.cfg.stopFor.Seconds())) + } + log.Printf("fault: %s", command) + if _, _, err := d.ctl.run(command); err != nil { + d.fail("fault %s: %v", command, err) + return + } + if err := d.connect(n); err != nil { + d.fail("fault %s: reconnect: %v", command, err) + return + } + n.up.Store(false) + d.faults.Add(1) + d.logRecord(record{Type: "fault", Fault: kind, Victim: n.id, Op: Op{At: start.UnixMilli(), Ms: time.Since(start).Milliseconds()}}) + if err := d.verifyAll(n, true); err != nil { + d.fail("after fault %s: node %d: %v", command, n.id, err) + return + } + n.up.Store(true) +} + +// quiescedFsck stops all operations, unmounts every node, runs fsck, and mounts again. +func (d *driver) quiescedFsck() { + d.pause.Lock() + defer d.pause.Unlock() + d.faultMu.Lock() + defer d.faultMu.Unlock() + start := time.Now() + for _, n := range d.nodes { + n.up.Store(false) + n.agent.close() + } + if _, _, err := d.ctl.run("unmount-all"); err != nil { + d.fail("fsck: unmount: %v", err) + return + } + rc, lines, err := d.ctl.run("fsck") + if err != nil { + d.fail("fsck: %v", err) + return + } + var leaks, other []string + for _, line := range lines { + switch { + case strings.Contains(line, "is an orphan"): + case isLeak(line): + leaks = append(leaks, line) + default: + other = append(other, line) + } + } + d.fscks.Add(1) + d.logRecord(record{Type: "fsck", Detail: fmt.Sprintf("%s, %d leaks, %d other", rc, len(leaks), len(other)), Op: Op{At: start.UnixMilli()}}) + if len(other) > 0 { + d.fail("fsck found more than leaks:\n%s", strings.Join(other, "\n")) + return + } + if len(leaks) > 0 { + log.Printf("fsck: repairing %d leaks", len(leaks)) + if _, _, err := d.ctl.run("fsck -repair"); err != nil { + d.fail("fsck -repair: %v", err) + return + } + if _, lines, err = d.ctl.run("fsck"); err != nil || slices.ContainsFunc(lines, func(l string) bool { return !strings.Contains(l, "is an orphan") }) { + d.fail("fsck after repair: %v %v", err, lines) + return + } + } + if _, _, err := d.ctl.run("mount-all"); err != nil { + d.fail("fsck: mount: %v", err) + return + } + for _, n := range d.nodes { + if err := d.connect(n); err != nil { + d.fail("fsck: reconnect node %d: %v", n.id, err) + return + } + } + for _, n := range d.nodes { + if err := d.verifyAll(n, true); err != nil { + d.fail("after fsck: node %d: %v", n.id, err) + return + } + } + log.Printf("fsck: %d leaks repaired, every node shows the model (%s)", len(leaks), time.Since(start).Round(time.Second)) +} + +func isLeak(line string) bool { + for _, s := range []string{"leaked", "will never be freed", "unnamed: no directory names it", "belongs to no snapshot"} { + if strings.Contains(line, s) { + return true + } + } + return false +} + +// barrier pauses every lane and checks every node lists every directory as the model does. +func (d *driver) barrier() { + d.pause.Lock() + defer d.pause.Unlock() + for _, n := range d.upNodes() { + if err := d.onNode(n, func() error { return d.verifyAll(n, false) }); err != nil { + d.fail("barrier: node %d: %v", n.id, err) + return + } + } + d.barriers.Add(1) +} + +func (d *driver) report() string { + d.model.mu.Lock() + files, snaps, blocks := len(d.model.Files), len(d.model.Snaps), d.model.blocks() + d.model.mu.Unlock() + var kinds []string + d.opsByKind.Range(func(k, v any) bool { + kinds = append(kinds, fmt.Sprintf("%s=%d", k, v.(*atomic.Int64).Load())) + return true + }) + sort.Strings(kinds) + return fmt.Sprintf("ops=%d unexpected=%d faults=%d fscks=%d barriers=%d slow_checks=%d slowest_converge_ms=%d files=%d snapshots=%d blocks=%d [%s]", + d.ops.Load(), d.unexpected.Load(), d.faults.Load(), d.fscks.Load(), d.barriers.Load(), d.slow.Load(), d.slowestMs.Load(), + files, snaps, blocks, strings.Join(kinds, " ")) +} + +func newDriver(cfg config) (*driver, func(), error) { + if err := os.MkdirAll(cfg.out, 0o755); err != nil { + return nil, nil, err + } + logf, err := os.Create(filepath.Join(cfg.out, "ops.jsonl")) + if err != nil { + return nil, nil, err + } + clusterLog, err := os.Create(filepath.Join(cfg.out, "cluster.log")) + if err != nil { + return nil, nil, err + } + env := []string{"NODE_COUNT=3", fmt.Sprintf("VM_SSH_BASE_PORT=%d", cfg.sshBasePort), fmt.Sprintf("VM_CLUSTER_SIZE_MB=%d", cfg.diskMB)} + ctl, err := startController(filepath.Join(cfg.checkout, "tests/soak/cluster.sh"), env, clusterLog) + if err != nil { + return nil, nil, err + } + ctx, cancel := context.WithCancel(context.Background()) + d := &driver{cfg: cfg, ctl: ctl, model: newModel(), logw: logf, stderr: clusterLog, ctx: ctx, cancel: cancel} + cleanup := func() { + cancel() + for _, n := range d.nodes { + if n.agent != nil { + n.agent.close() + } + } + ctl.stop() + _ = logf.Close() + _ = clusterLog.Close() + } + return d, cleanup, nil +} + +func every(ctx context.Context, interval time.Duration, f func()) { + if interval <= 0 { + return + } + t := time.NewTicker(interval) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + f() + } + } +} + +func run(cfg config) error { + d, cleanup, err := newDriver(cfg) + if err != nil { + return err + } + defer cleanup() + log.Printf("soak: seed %d, %d lanes, %s, faults every %s, fsck every %s, output in %s", + cfg.seed, cfg.lanes, cfg.duration, cfg.faultEvery, cfg.fsckEvery, cfg.out) + if err := d.bringUp(); err != nil { + return fmt.Errorf("bring up the cluster: %w", err) + } + ctx, stop := context.WithTimeout(d.ctx, cfg.duration) + defer stop() + faultRng := rand.New(rand.NewPCG(cfg.seed, 0xfa)) + var wg sync.WaitGroup + for i := range cfg.lanes { + l := &lane{id: i, rng: rand.New(rand.NewPCG(cfg.seed, uint64(i))), budget: cfg.budget} + wg.Go(func() { d.runLane(l) }) + } + wg.Go(func() { + every(ctx, cfg.faultEvery, func() { + d.injectFault(pick(faultRng, []string{"power", "sigkill", "stop"}), pick(faultRng, d.nodes)) + }) + }) + wg.Go(func() { every(ctx, cfg.fsckEvery, d.quiescedFsck) }) + wg.Go(func() { every(ctx, cfg.barrierEvery, d.barrier) }) + wg.Go(func() { every(ctx, time.Minute, func() { log.Print(d.report()) }) }) + <-ctx.Done() + d.stopping.Store(true) + d.cancel() + wg.Wait() + log.Print(d.report()) + return d.finish() +} + +// finish writes the outcome and, after a failure, keeps the shared disk for analysis. +func (d *driver) finish() error { + if d.failure == "" { + return os.WriteFile(filepath.Join(d.cfg.out, "result.txt"), []byte("PASS "+d.report()+"\n"), 0o644) + } + text := "FAIL " + d.failure + "\n" + d.report() + "\n" + if err := os.WriteFile(filepath.Join(d.cfg.out, "result.txt"), []byte(text), 0o644); err != nil { + return err + } + if _, _, err := d.ctl.run("save-image " + filepath.Join(d.cfg.out, "cluster.img")); err != nil { + log.Printf("save the shared disk: %v", err) + } + return errors.New(d.failure) +} diff --git a/tests/soak/main.go b/tests/soak/main.go new file mode 100644 index 00000000..88761f3a --- /dev/null +++ b/tests/soak/main.go @@ -0,0 +1,127 @@ +//go:build linux + +// Command goclusterfs-soak runs a long random workload on a 3-VM cluster, checks every node against a model of +// the expected state, and cuts power, kills or stops nodes on the way. See README.md. +package main + +import ( + "bufio" + "encoding/json" + "errors" + "flag" + "fmt" + "log" + "os" + "path/filepath" + "time" +) + +func main() { + log.SetFlags(log.LstdFlags | log.Lmicroseconds) + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "usage: goclusterfs-soak run|replay|agent [flags]") + os.Exit(2) + } + var err error + switch os.Args[1] { + case "agent": + fs := flag.NewFlagSet("agent", flag.ExitOnError) + root := fs.String("root", mountPoint, "mount point to run operations under") + _ = fs.Parse(os.Args[2:]) + err = runAgent(*root, os.Stdin, os.Stdout) + case "run", "replay": + var cfg config + var oplog string + var upto uint64 + fs := flag.NewFlagSet(os.Args[1], flag.ExitOnError) + fs.StringVar(&cfg.checkout, "checkout", ".", "repository checkout with built goclusterfs-mount and goclusterfs-fsck") + fs.StringVar(&cfg.out, "out", filepath.Join("soak-out", time.Now().Format("20060102-150405")), "directory for the op log, cluster log and result") + fs.DurationVar(&cfg.duration, "duration", 48*time.Hour, "how long to run") + fs.IntVar(&cfg.lanes, "lanes", 3, "concurrent operation streams, each on its own files in the shared directories") + fs.Uint64Var(&cfg.seed, "seed", uint64(time.Now().UnixNano()), "random seed") + fs.DurationVar(&cfg.faultEvery, "fault-every", 10*time.Minute, "interval between faults; 0 for none") + fs.DurationVar(&cfg.fsckEvery, "fsck-every", time.Hour, "interval between quiesced fsck runs; 0 for none") + fs.DurationVar(&cfg.barrierEvery, "barrier-every", 5*time.Minute, "interval between full directory checks; 0 for none") + fs.DurationVar(&cfg.window, "window", 30*time.Second, "how long a node may take to show an acknowledged change, including waiting out a failed node") + fs.DurationVar(&cfg.coherence, "coherence", 6*time.Second, "the documented coherence window; slower checks are logged") + fs.DurationVar(&cfg.stopFor, "stop-for", 25*time.Second, "how long a stop fault keeps the mount stopped; past the 10s heartbeat timeout") + fs.DurationVar(&cfg.opTimeout, "op-timeout", 5*time.Minute, "an operation taking longer is a hang") + fs.IntVar(&cfg.sshBasePort, "ssh-base-port", 3122, "host SSH port of VM 1; keep it apart from other VM runs") + fs.IntVar(&cfg.diskMB, "disk-mb", 4096, "shared disk size") + fs.Int64Var(&cfg.budget, "budget", 1200, "data blocks the model may hold, counting each snapshot as a full copy") + fs.StringVar(&oplog, "log", "", "replay: op log to run again") + fs.Uint64Var(&upto, "upto", 0, "replay: stop after this op sequence number") + _ = fs.Parse(os.Args[2:]) + if os.Args[1] == "run" { + err = run(cfg) + } else { + err = replay(cfg, oplog, upto) + } + default: + err = fmt.Errorf("unknown command %q", os.Args[1]) + } + if err != nil { + log.Fatal(err) + } +} + +// replay runs a logged soak again in log order, with the same operations, nodes, data and faults. +func replay(cfg config, oplog string, upto uint64) error { + if oplog == "" { + return errors.New("replay needs -log") + } + f, err := os.Open(oplog) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + var records []record + sc := bufio.NewScanner(f) + sc.Buffer(make([]byte, 1<<20), 1<<20) + for sc.Scan() { + var r record + if err := json.Unmarshal(sc.Bytes(), &r); err != nil { + return fmt.Errorf("%s: %w", oplog, err) + } + records = append(records, r) + } + if err := sc.Err(); err != nil { + return err + } + + d, cleanup, err := newDriver(cfg) + if err != nil { + return err + } + defer cleanup() + if err := d.bringUp(); err != nil { + return fmt.Errorf("bring up the cluster: %w", err) + } + for _, r := range records { + if d.ctx.Err() != nil || upto > 0 && r.Type == "op" && r.Seq > upto { + break + } + switch r.Type { + case "op": + op := r.Op + logged := op.Want + op.Got, op.Err = 0, "" + d.model.mu.Lock() + op.Want = d.model.expect(&op) + d.model.mu.Unlock() + if op.Want != logged { + d.fail("op %d: the model expects errno %d where the run expected %d", op.Seq, op.Want, logged) + break + } + n := d.nodes[op.Node-1] + n.mu.RLock() + d.runOpLocked(&op, n) + case "fault": + d.injectFault(r.Fault, d.nodes[r.Victim-1]) + case "fsck": + d.quiescedFsck() + } + } + log.Print(d.report()) + return d.finish() +} diff --git a/tests/soak/model.go b/tests/soak/model.go new file mode 100644 index 00000000..009d31d4 --- /dev/null +++ b/tests/soak/model.go @@ -0,0 +1,428 @@ +//go:build linux + +package main + +import ( + "bytes" + "fmt" + "math/rand/v2" + "path" + "slices" + "strings" + "sync" + "syscall" +) + +const ( + blockSize = 1 << 20 + maxFile = 8 << 20 + // anyErrno is the expected result of an operation that must fail, with any errno. + anyErrno = -2 +) + +// sharedDirs hold files of every lane and are never removed. +var sharedDirs = []string{"/soak/d0", "/soak/d1", "/soak/d2", "/soak/d3"} + +// Op is one operation as the driver runs and logs it. +type Op struct { + Seq uint64 `json:"seq"` + Lane int `json:"lane"` + Node int `json:"node"` + Kind string `json:"op"` + Path string `json:"path,omitempty"` + Path2 string `json:"path2,omitempty"` + Name string `json:"name,omitempty"` + Off int64 `json:"off,omitempty"` + Len int64 `json:"len,omitempty"` + Seed uint64 `json:"seed,omitempty"` + Want int `json:"want"` + Got int `json:"got"` + Err string `json:"err,omitempty"` + At int64 `json:"at"` + Ms int64 `json:"ms"` +} + +func (op *Op) request() request { + r := request{Op: op.Kind, Path: op.Path, Path2: op.Path2, Off: op.Off, Len: op.Len, Seed: op.Seed} + switch op.Kind { + case "snapshot": + r.Op, r.Attr, r.Value = "setxattr", "user.snapshot", op.Name + case "clone": + r.Op, r.Attr, r.Value = "setxattr", "user.clone", op.Path2 + case "snapclone": + r.Op, r.Attr, r.Value = "setxattr", "user.snapshot.clone", op.Name+":"+op.Path2 + case "snapdelete": + r.Op, r.Attr, r.Value = "setxattr", "user.snapshot.delete", op.Name + } + return r +} + +// File is a file in the model. Its pointer is its identity: a later file at the same path is another File. +type File struct { + Data []byte + Snaps map[string]bool +} + +// Snap is a snapshot. Owner is nil once its file is gone, which makes it an orphan. +type Snap struct { + Data []byte + Owner *File +} + +// Model is the state every node must show once an acknowledged operation is visible. +type Model struct { + mu sync.Mutex + Files map[string]*File + Dirs map[string]bool + Snaps map[string]*Snap +} + +func newModel() *Model { + m := &Model{Files: map[string]*File{}, Dirs: map[string]bool{"/": true, "/soak": true}, Snaps: map[string]*Snap{}} + for _, d := range sharedDirs { + m.Dirs[d] = true + } + return m +} + +func (m *Model) exists(p string) bool { return m.Files[p] != nil || m.Dirs[p] } + +func (m *Model) hasChildren(dir string) bool { + for p := range m.Files { + if path.Dir(p) == dir { + return true + } + } + for p := range m.Dirs { + if p != dir && path.Dir(p) == dir { + return true + } + } + return false +} + +// expect returns the errno the operation must return on the model's current state. +func (m *Model) expect(op *Op) int { + f := m.Files[op.Path] + switch op.Kind { + case "create", "mkdir": + switch { + case m.exists(op.Path): + return int(syscall.EEXIST) + case !m.Dirs[path.Dir(op.Path)]: + return int(syscall.ENOENT) + } + case "write", "truncate", "unlink": + switch { + case m.Dirs[op.Path]: + return int(syscall.EISDIR) + case f == nil: + return int(syscall.ENOENT) + } + case "rmdir": + switch { + case f != nil: + return int(syscall.ENOTDIR) + case !m.Dirs[op.Path]: + return int(syscall.ENOENT) + case m.hasChildren(op.Path): + return int(syscall.ENOTEMPTY) + } + case "rename": + return anyErrno + case "snapshot": + switch { + case f == nil: + return int(syscall.ENOENT) + case m.Snaps[op.Name] != nil: + return int(syscall.EEXIST) + } + case "clone", "snapclone": + switch { + case f == nil: + return int(syscall.ENOENT) + case op.Kind == "snapclone" && (m.Snaps[op.Name] == nil || m.Snaps[op.Name].Owner != f): + return int(syscall.ENOENT) + case m.exists(op.Path2): + return int(syscall.EEXIST) + } + case "snapdelete": + if f == nil || m.Snaps[op.Name] == nil || m.Snaps[op.Name].Owner != f { + return int(syscall.ENOENT) + } + } + return 0 +} + +// apply changes the model as a successful operation changed the filesystem. +func (m *Model) apply(op *Op) { + f := m.Files[op.Path] + switch op.Kind { + case "create": + m.Files[op.Path] = &File{Snaps: map[string]bool{}} + case "write": + data := genData(op.Seed, op.Len) + if end := op.Off + op.Len; end > int64(len(f.Data)) { + f.Data = append(f.Data, make([]byte, end-int64(len(f.Data)))...) + } + copy(f.Data[op.Off:], data) + case "truncate": + if op.Len < int64(len(f.Data)) { + f.Data = f.Data[:op.Len] + } else { + f.Data = append(f.Data, make([]byte, op.Len-int64(len(f.Data)))...) + } + case "unlink": + for name := range f.Snaps { + m.Snaps[name].Owner = nil + } + delete(m.Files, op.Path) + case "mkdir": + m.Dirs[op.Path] = true + case "rmdir": + delete(m.Dirs, op.Path) + case "snapshot": + m.Snaps[op.Name] = &Snap{Data: bytes.Clone(f.Data), Owner: f} + f.Snaps[op.Name] = true + case "clone": + m.Files[op.Path2] = &File{Data: bytes.Clone(f.Data), Snaps: map[string]bool{}} + case "snapclone": + m.Files[op.Path2] = &File{Data: bytes.Clone(m.Snaps[op.Name].Data), Snaps: map[string]bool{}} + case "snapdelete": + delete(m.Snaps, op.Name) + delete(f.Snaps, op.Name) + } +} + +// blocks estimates the data blocks the model holds, counting each snapshot as a full copy. +func (m *Model) blocks() int64 { + var n int64 + for _, f := range m.Files { + n += (int64(len(f.Data)) + blockSize - 1) / blockSize + } + for _, s := range m.Snaps { + n += (int64(len(s.Data)) + blockSize - 1) / blockSize + } + return n +} + +// lane generates operations on its own files, directories and snapshots, whose names start with its prefix. +type lane struct { + id int + rng *rand.Rand + next int + budget int64 +} + +func (l *lane) prefix() string { return fmt.Sprintf("l%d-", l.id) } + +func (l *lane) name(kind string) string { + l.next++ + return fmt.Sprintf("%s%s%d", l.prefix(), kind, l.next) +} + +func (l *lane) owns(p string) bool { return strings.HasPrefix(path.Base(p), l.prefix()) } + +func pick[T any](r *rand.Rand, xs []T) T { return xs[r.IntN(len(xs))] } + +// generate picks the lane's next operation from the model's current state. The caller holds m.mu. +func (l *lane) generate(m *Model) *Op { + var files, dirs, snaps []string + orphans := 0 + for p := range m.Files { + if l.owns(p) { + files = append(files, p) + } + } + for p := range m.Dirs { + if l.owns(p) { + dirs = append(dirs, p) + } + } + for name, s := range m.Snaps { + switch { + case !l.owns(name): + case s.Owner != nil: + snaps = append(snaps, name) + default: + orphans++ + } + } + slices.Sort(files) + slices.Sort(dirs) + slices.Sort(snaps) + parents := append(slices.Clone(sharedDirs), dirs...) + full := m.blocks() > l.budget + r := l.rng + + weights := []struct { + kind string + w int + }{ + {"create", 15}, {"write", 30}, {"truncate", 8}, {"unlink", 8}, {"mkdir", 4}, {"rmdir", 4}, + {"rename", 2}, {"snapshot", 6}, {"clone", 3}, {"snapclone", 3}, {"snapdelete", 6}, {"invalid", 3}, + } + total := 0 + for _, w := range weights { + total += w.w + } + n := r.IntN(total) + kind := "" + for _, w := range weights { + if n -= w.w; n < 0 { + kind = w.kind + break + } + } + if full && slices.Contains([]string{"create", "write", "snapshot", "clone", "snapclone"}, kind) { + kind = pick(r, []string{"unlink", "truncate", "snapdelete"}) + } + if len(files) == 0 || len(files) < 6 && kind != "create" && kind != "mkdir" { + kind = "create" + } + if len(files) > 40 && kind == "create" { + kind = "unlink" + } + + op := &Op{Lane: l.id, Kind: kind} + switch kind { + case "create": + op.Path = path.Join(pick(r, parents), l.name("f")) + case "write": + op.Path = pick(r, files) + size := int64(len(m.Files[op.Path].Data)) + op.Len = pickLen(r) + switch v := r.IntN(10); { + case v < 5 && size > 0: + op.Off = r.Int64N(size) + case v < 8: + op.Off = size + default: + op.Off = size + r.Int64N(2*blockSize) + } + if op.Off+op.Len > maxFile { + op.Off, op.Len = 0, min(op.Len, 64<<10) + } + op.Seed = r.Uint64() + case "truncate": + op.Path = pick(r, files) + size := int64(len(m.Files[op.Path].Data)) + if r.IntN(2) == 0 || full { + op.Len = r.Int64N(size + 1) + } else { + op.Len = min(size+r.Int64N(blockSize), maxFile) + } + case "unlink": + op.Path = pick(r, files) + if f := m.Files[op.Path]; len(f.Snaps) > 0 && (orphans >= 20 || r.IntN(10) > 0) { + op.Kind, op.Name = "snapdelete", pickKey(r, f.Snaps) + } + case "mkdir": + if len(dirs) >= 6 { + op.Kind, op.Path = "rmdir", pick(r, dirs) + } else { + op.Path = path.Join(pick(r, sharedDirs), l.name("d")) + } + case "rmdir": + if len(dirs) == 0 { + op.Kind, op.Path = "mkdir", path.Join(pick(r, sharedDirs), l.name("d")) + break + } + op.Path = pick(r, dirs) + if m.hasChildren(op.Path) && r.IntN(2) == 0 { + for _, p := range files { + if path.Dir(p) == op.Path { + op.Kind, op.Path = "unlink", p + break + } + } + } + case "rename": + op.Path = pick(r, files) + op.Path2 = path.Join(path.Dir(op.Path), l.name("r")) + case "snapshot": + op.Path, op.Name = pick(r, files), l.name("s") + case "clone": + op.Path, op.Path2 = pick(r, files), "/"+l.name("c") + case "snapclone", "snapdelete": + if len(snaps) == 0 { + op.Kind, op.Path, op.Name = "snapshot", pick(r, files), l.name("s") + break + } + op.Name = pick(r, snaps) + op.Path = m.pathOf(m.Snaps[op.Name].Owner) + if kind == "snapclone" { + op.Path2 = "/" + l.name("c") + } + case "invalid": + switch r.IntN(3) { + case 0: + op.Kind, op.Path = "create", pick(r, files) + case 1: + op.Kind, op.Path, op.Off, op.Len, op.Seed = "write", path.Join(pick(r, parents), l.name("missing")), 0, 4096, r.Uint64() + default: + op.Kind, op.Path, op.Name, op.Path2 = "snapclone", pick(r, files), l.name("nosnap"), "/"+l.name("c") + } + } + return op +} + +func pickLen(r *rand.Rand) int64 { + switch v := r.IntN(20); { + case v < 14: + return 1 + r.Int64N(16<<10) + case v < 19: + return 64<<10 + r.Int64N(blockSize) + default: + return blockSize + r.Int64N(2*blockSize) + } +} + +func pickKey(r *rand.Rand, m map[string]bool) string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + slices.Sort(keys) + return pick(r, keys) +} + +func (m *Model) pathOf(f *File) string { + for p, g := range m.Files { + if g == f { + return p + } + } + return "" +} + +// snapsOf returns the names of the snapshots a node must list for the file at p. +func (m *Model) snapsOf(p string) []string { + f := m.Files[p] + if f == nil { + return nil + } + names := make([]string, 0, len(f.Snaps)) + for name := range f.Snaps { + names = append(names, name) + } + slices.Sort(names) + return names +} + +// listing returns what a node must list in dir, as the agent describes entries. +func (m *Model) listing(dir string) []string { + var out []string + for p, f := range m.Files { + if path.Dir(p) == dir { + out = append(out, fmt.Sprintf("f:%s:%d", path.Base(p), len(f.Data))) + } + } + for p := range m.Dirs { + if p != dir && path.Dir(p) == dir { + out = append(out, "d:"+path.Base(p)) + } + } + slices.Sort(out) + return out +} diff --git a/tests/soak/model_test.go b/tests/soak/model_test.go new file mode 100644 index 00000000..74f728fa --- /dev/null +++ b/tests/soak/model_test.go @@ -0,0 +1,70 @@ +//go:build linux + +package main + +import ( + "math/rand/v2" + "strings" + "syscall" + "testing" +) + +func step(t *testing.T, m *Model, op *Op, want int) { + t.Helper() + if got := m.expect(op); got != want { + t.Fatalf("%s %s: expect %d, want %d", op.Kind, op.Path, got, want) + } + if want == 0 { + m.apply(op) + } +} + +func TestModelFollowsTheFilesystemRules(t *testing.T) { + m := newModel() + f := "/soak/d0/l0-f1" + step(t, m, &Op{Kind: "create", Path: f}, 0) + step(t, m, &Op{Kind: "create", Path: f}, int(syscall.EEXIST)) + step(t, m, &Op{Kind: "write", Path: f, Off: 10, Len: 5, Seed: 7}, 0) + if data := m.Files[f].Data; len(data) != 15 || data[0] != 0 || string(data[10:]) != string(genData(7, 5)) { + t.Fatalf("write at 10 of an empty file left %v", data) + } + step(t, m, &Op{Kind: "snapshot", Path: f, Name: "l0-s1"}, 0) + step(t, m, &Op{Kind: "snapshot", Path: f, Name: "l0-s1"}, int(syscall.EEXIST)) + step(t, m, &Op{Kind: "mkdir", Path: "/soak/d0/l0-d1"}, 0) + step(t, m, &Op{Kind: "create", Path: "/soak/d0/l0-d1/l0-f2"}, 0) + step(t, m, &Op{Kind: "rmdir", Path: "/soak/d0/l0-d1"}, int(syscall.ENOTEMPTY)) + step(t, m, &Op{Kind: "unlink", Path: f}, 0) + if m.Snaps["l0-s1"].Owner != nil { + t.Fatal("the deleted file's snapshot still has an owner") + } + step(t, m, &Op{Kind: "create", Path: f}, 0) + step(t, m, &Op{Kind: "snapclone", Path: f, Name: "l0-s1", Path2: "/l0-c1"}, int(syscall.ENOENT)) + step(t, m, &Op{Kind: "snapdelete", Path: f, Name: "l0-s1"}, int(syscall.ENOENT)) + step(t, m, &Op{Kind: "rename", Path: f, Path2: "/soak/d0/l0-r1"}, anyErrno) +} + +// A lane only names and touches its own files, directories and snapshots, the same way for the same seed. +func TestLaneStaysInItsNamesAndRepeats(t *testing.T) { + gen := func() []string { + m := newModel() + l := &lane{id: 1, rng: rand.New(rand.NewPCG(42, 1)), budget: 1000} + var trace []string + for range 2000 { + op := l.generate(m) + for _, p := range []string{op.Path, op.Path2, op.Name} { + if p != "" && !strings.Contains(p, "/l1-") && !strings.HasPrefix(p, "l1-") { + t.Fatalf("%s names %q, outside lane 1", op.Kind, p) + } + } + if op.Kind != "rename" && m.expect(op) == 0 { + m.apply(op) + } + trace = append(trace, op.Kind+" "+op.Path) + } + return trace + } + a, b := gen(), gen() + if strings.Join(a, "\n") != strings.Join(b, "\n") { + t.Fatal("the same seed generated different operations") + } +}