Skip to content
Merged
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
/goclusterfs
/goclusterfs-mount
/goclusterfs-fsck
/goclusterfs-soak
/soak-out/
/gfsctl
/GoClusterFS
/coverage.out
Expand Down
8 changes: 6 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# GoClusterFS Makefile
# Build and test automation for the distributed filesystem

.PHONY: build build-mount build-fsck build-gfsctl completions dashboard test test-unit test-integration test-all test-parity test-multinode-docker docker-build-test docker-clean-test test-multinode-vm vm-clean clean fmt vet check-fuse check-qemu lint vuln help
.PHONY: build build-mount build-fsck build-gfsctl soak completions dashboard test test-unit test-integration test-all test-parity test-multinode-docker docker-build-test docker-clean-test test-multinode-vm vm-clean clean fmt vet check-fuse check-qemu lint vuln help

# Go parameters
GOCMD=go
Expand All @@ -22,7 +22,7 @@ DASHBOARD_DIST=internal/webui/build/dist
GOLANGCI_LINT_VERSION=v2.15.0
GOVULNCHECK_VERSION=v1.9.0
COVER_PROFILE=coverage.out
UNIT_PKGS ?= ./internal/...
UNIT_PKGS ?= ./internal/... ./tests/soak
UNIT_TIMEOUT ?= 20m

# Default target
Expand All @@ -41,6 +41,10 @@ build-fsck:
$(GOBUILD) -o $(FSCK_BINARY) ./cmd/goclusterfs-fsck
@echo "Built $(FSCK_BINARY) successfully"

# Static, so the same binary runs on the host as the driver and in the Alpine VMs as the agent
soak: build-mount build-fsck
CGO_ENABLED=0 $(GOBUILD) -o goclusterfs-soak ./tests/soak

build-gfsctl:
@echo "Building $(GFSCTL_BINARY)..."
$(GOBUILD) -o $(GFSCTL_BINARY) ./cmd/gfsctl
Expand Down
2 changes: 1 addition & 1 deletion tests/scripts/unit-groups.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ default_seconds=5

list_packages() {
local out
out=$(go list ./internal/...)
out=$(go list ./internal/... ./tests/soak)
echo "$out"
}

Expand Down
39 changes: 39 additions & 0 deletions tests/soak/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
# Multi-node soak

`goclusterfs-soak` runs a random mix of operations on a 3-VM cluster for hours or days and checks every node against a model of the expected state.

- **Operations:** create, write (overwrite, append, sparse), truncate, unlink, mkdir, rmdir, rename, snapshot, clone, clone from a snapshot, and snapshot delete. Rename is not supported, so each rename must fail and change nothing.
- **Lanes:** several lanes run at once, each on its own files and snapshots, in directories they share.
- **Checks:** after every operation, each node must show its result, through a fresh open, within `-window` (30s).
- The window covers `-attr-timeout` and `-cache-ttl`, because broadcast cannot cross the VMs' separate networks.
- It also covers waiting out a failed node's 10s detection.
- A check that needs longer than `-coherence` (6s) is logged as a `slow` record for review.
- **Faults:** every `-fault-every`, one node is hit, picked at random from these:
- power is cut (QEMU is killed);
- goclusterfs-mount gets SIGKILL;
- goclusterfs-mount is stopped past the 10s heartbeat timeout and resumed.

The node comes back, mounts again and must show the model. The other nodes keep working through the fault.
- **fsck:** every `-fsck-every`, all operations stop, every node unmounts and goclusterfs-fsck runs. Anything other than leaks and orphan snapshots is a failure. Leaks are repaired.

The first failure stops the run. `result.txt` says what failed, `ops.jsonl` logs every operation, fault and fsck, and `cluster.img` keeps the shared disk.

## Run

On a host with KVM, from a checkout of its own:

```bash
make soak
flock /tmp/goclusterfs-soak.lock systemd-run --scope -p MemoryMax=8G -p CPUQuota=400% --quiet \
./goclusterfs-soak run -duration 48h -out /root/gcfs-soak/run1
```

The VMs use SSH ports from `-ssh-base-port` (3122), so the soak can share a host with other VM test runs. It logs a summary every minute.

## Replay

```bash
./goclusterfs-soak replay -log /root/gcfs-soak/run1/ops.jsonl -out /root/gcfs-soak/replay1
```

Replay boots a new cluster and runs the logged operations one at a time, in log order, on the same nodes, with the same data, faults and fsck runs. `-upto N` stops after operation N. Lanes ran at the same time in the original run, so a failure that needs their exact timing may not happen again.
174 changes: 174 additions & 0 deletions tests/soak/agent.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
//go:build linux

package main

import (
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"math/rand/v2"
"os"
"path/filepath"
"sync"
"syscall"

"golang.org/x/sys/unix"
)

// request is one operation the driver asks an agent to run. Paths are relative to the mount.
type request struct {
ID uint64 `json:"id"`
Op string `json:"op"`
Path string `json:"path,omitempty"`
Path2 string `json:"path2,omitempty"`
Attr string `json:"attr,omitempty"`
Value string `json:"value,omitempty"`
Off int64 `json:"off,omitempty"`
Len int64 `json:"len,omitempty"`
Seed uint64 `json:"seed,omitempty"`
}

type response struct {
ID uint64 `json:"id"`
Errno int `json:"errno"`
Err string `json:"err,omitempty"`
Size int64 `json:"size,omitempty"`
Sum string `json:"sum,omitempty"`
Names []string `json:"names,omitempty"`
Text string `json:"text,omitempty"`
}

// genData returns n bytes that depend only on seed, so the driver and the agents produce the same data.
func genData(seed uint64, n int64) []byte {
b := make([]byte, n)
r := rand.New(rand.NewPCG(seed, seed^0x9e3779b97f4a7c15))
for i := int64(0); i < n; i += 8 {
v := r.Uint64()
for j := int64(0); j < 8 && i+j < n; j++ {
b[i+j] = byte(v >> (8 * j))
}
}
return b
}

func sum(b []byte) string {
h := sha256.Sum256(b)
return hex.EncodeToString(h[:8])
}

// runAgent serves requests from stdin until it closes, running each in its own goroutine.
func runAgent(root string, in io.Reader, out io.Writer) error {
dec := json.NewDecoder(in)
enc := json.NewEncoder(out)
var mu sync.Mutex
var wg sync.WaitGroup
defer wg.Wait()
for {
var req request
if err := dec.Decode(&req); err != nil {
if errors.Is(err, io.EOF) {
return nil
}
return err
}
wg.Add(1)
go func() {
defer wg.Done()
resp := serve(root, req)
resp.ID = req.ID
mu.Lock()
defer mu.Unlock()
if err := enc.Encode(resp); err != nil {
fmt.Fprintf(os.Stderr, "agent: reply %d: %v\n", req.ID, err)
}
}()
}
}

func serve(root string, req request) response {
var resp response
p := filepath.Join(root, req.Path)
var err error
switch req.Op {
case "ping":
case "create":
var f *os.File
if f, err = os.OpenFile(p, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644); err == nil {
err = f.Close()
}
case "write":
err = writeSynced(p, req.Off, genData(req.Seed, req.Len))
case "truncate":
err = unix.Truncate(p, req.Len)
case "unlink":
err = unix.Unlink(p)
case "mkdir":
err = unix.Mkdir(p, 0o755)
case "rmdir":
err = unix.Rmdir(p)
case "rename":
err = unix.Rename(p, filepath.Join(root, req.Path2))
case "setxattr":
err = unix.Setxattr(p, req.Attr, []byte(req.Value), 0)
case "getxattr":
buf := make([]byte, 64<<10)
var n int
if n, err = unix.Getxattr(p, req.Attr, buf); err == nil {
resp.Text = string(buf[:n])
}
case "read":
var data []byte
if data, err = os.ReadFile(p); err == nil {
resp.Size, resp.Sum = int64(len(data)), sum(data)
}
case "list":
var entries []os.DirEntry
if entries, err = os.ReadDir(p); err == nil {
for _, e := range entries {
resp.Names = append(resp.Names, entryName(p, e))
}
}
default:
err = fmt.Errorf("unknown op %q", req.Op)
}
if err != nil {
resp.Err = err.Error()
resp.Errno = -1
var errno syscall.Errno
if errors.As(err, &errno) {
resp.Errno = int(errno)
}
}
return resp
}

// entryName describes a directory entry as "d:name" or "f:name:size".
func entryName(dir string, e os.DirEntry) string {
if e.IsDir() {
return "d:" + e.Name()
}
var st unix.Stat_t
if err := unix.Lstat(filepath.Join(dir, e.Name()), &st); err != nil {
return "f:" + e.Name() + ":?"
}
return fmt.Sprintf("f:%s:%d", e.Name(), st.Size)
}

func writeSynced(p string, off int64, data []byte) error {
f, err := os.OpenFile(p, os.O_WRONLY, 0)
if err != nil {
return err
}
if _, err := f.WriteAt(data, off); err != nil {
_ = f.Close()
return err
}
if err := f.Sync(); err != nil {
_ = f.Close()
return err
}
return f.Close()
}
Loading
Loading