From 64c07a53aaac824ec4be684e7036ca0b82c68fa1 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 22:38:45 -0600 Subject: [PATCH 1/2] fix(snapshot): keep each snapshot with the file it was taken of Snapshots matched files by inode number alone, so a new file that reused a deleted file's inode slot listed its snapshots, could delete them, and could clone them, reading the deleted file's data. Inodes now record a birth generation when they take a slot, above every generation the slot's earlier files reached, and each snapshot records its file's. Listing, clone and delete through a file, and the API's file listing, match number and birth. Snapshots of deleted files, and those older versions wrote without a birth, are orphans: the API's full listing marks them, a delete without a path removes them, and fsck warns. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 1 + CLAUDE.md | 2 +- cmd/mount/main.go | 6 +- docs/API-CONFIG.md | 5 +- docs/OPERATIONS.md | 2 + internal/api/writes.go | 2 + internal/fileops/fileops.go | 62 ++++++--- internal/fsck/fsck.go | 2 + internal/fsck/phase3_inodes.go | 4 + internal/fsck/phase7_snapshots.go | 27 +++- internal/goclusterfs/fs.go | 106 +++++++++++++++- internal/goclusterfs/snapshot_owner_test.go | 134 ++++++++++++++++++++ internal/inode/inode.go | 8 +- internal/snapshot/snapshot.go | 68 ++++++++-- internal/snapshot/snapshot_test.go | 31 +++++ tests/test-manifest.yaml | 28 ++++ 16 files changed, 441 insertions(+), 47 deletions(-) create mode 100644 internal/goclusterfs/snapshot_owner_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 155db63b..d3fe834f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Run `goclusterfs-fsck -repair` before the first mount after upgrading an image that has snapshots or clones. Unmount the device on every node first. - Older versions marked a file's blocks shared, on snapshot or clone, without counting the owners. Until `goclusterfs-fsck -repair` counts them, this version never frees such blocks, so deleting those files leaks their space, and it refuses to snapshot or clone them (`ENOTSUP`). In-place writes to an old clone still change its source. - Snapshots made by older versions captured no data. They still list and delete, but cloning one fails with `ENOTSUP`. +- Snapshots now record the birth generation of the file they were taken of, so a later file that reuses the inode number cannot list, clone or delete them. Snapshots older versions made do not record it, so after the upgrade they are orphans: `getfattr -n user.snapshots` no longer lists them, `GET /api/v1/snapshots` lists them with `orphan: true`, and `DELETE /api/v1/snapshots/` without `path` deletes them. Older binaries keep the birth generation in inode bytes 196–203 only while they do not rewrite the inode, so upgrade all nodes together. - `-skip-recovery` now takes the slot number from the recovery error, as `-skip-recovery=N`, and discards a journal only when the mount takes slot N. A bare `-skip-recovery` is a usage error. ### Added diff --git a/CLAUDE.md b/CLAUDE.md index 547d0226..301ed9bc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -90,7 +90,7 @@ Snapshots and clones work through extended attributes on a file (`internal/goclu - `setfattr -n user.clone -v / ` clones the file - `setfattr -n user.snapshot.clone -v :/ ` clones from a snapshot -Clone paths are absolute from the filesystem root, and clones can only go in the root directory. +Clone paths are absolute from the filesystem root, and clones can only go in the root directory. A snapshot records its file's inode number and birth generation, so a later file in the same inode slot does not see it; once its file is deleted it is an orphan, listed and deleted only through the API without a path. ## Current Limitations diff --git a/cmd/mount/main.go b/cmd/mount/main.go index 3b726d86..61270e92 100644 --- a/cmd/mount/main.go +++ b/cmd/mount/main.go @@ -502,7 +502,11 @@ func main() { broadcastOn: cacheMgr.BroadcastEnabled(), broadcastProblem: cacheMgr.BroadcastProblem, }) - apiServer.SetFileService(fileops.New(*mountPoint, gfs.SnapshotRegistry())) + var snaps fileops.Snapshots + if gfs.SnapshotRegistry() != nil { + snaps = gfs + } + apiServer.SetFileService(fileops.New(*mountPoint, snaps)) apiServer.SetAdminService(adminOps{journal: j, trim: trimMgr, cache: cacheMgr}) if err := apiServer.Start(ctx); err != nil { log.Printf("Warning: Failed to start the API server: %v; the mount continues without it. Choose a free -api-addr, or pass -api=false", err) diff --git a/docs/API-CONFIG.md b/docs/API-CONFIG.md index 07dc6ba8..c7c32fad 100644 --- a/docs/API-CONFIG.md +++ b/docs/API-CONFIG.md @@ -118,13 +118,14 @@ Each mount on a host needs its own `-api-addr`. When the port is taken, mount lo These endpoints change the filesystem or this node's state, and listing snapshots reads filesystem metadata. They only work with a token, also on loopback: the Host check stops web pages, not other local processes. Without a token they answer 403, and mount logs `API: snapshot, clone and admin endpoints are off`. The API refuses cross-origin writes: a CORS preflight for any method but `GET` or `HEAD` fails, and a write whose `Origin` header names another site gets 403. Same-origin pages and non-browser clients can call them. -Snapshots belong to single files. A request names the file by its path in the mount, such as `/dir/file`. The path must be absolute and clean, and may not go through a symlink or out of the mount. Snapshot names are unique in the filesystem. The API creates and clones only names without `:`, `/` or NUL, because `user.snapshot.clone` splits at the first `:`. It deletes any name; escape `/` as `%2F` in the URL. +Snapshots belong to single files. A snapshot stays with the file it was taken of: once that file is deleted, the snapshot is an orphan, which only the full listing shows and only a delete without `path` removes. A later file that reuses the inode number does not see it. Snapshots written by versions before this record cannot prove their file, so they are orphans too. A request names the file by its path in the mount, such as `/dir/file`. The path must be absolute and clean, and may not go through a symlink or out of the mount. Snapshot names are unique in the filesystem. The API creates and clones only names without `:`, `/` or NUL, because `user.snapshot.clone` splits at the first `:`. It deletes any name; escape `/` as `%2F` in the URL. | Method and path | Body | Does the same as | Success | |-----------------|------|------------------|---------| -| `GET /api/v1/snapshots[?path=/f]` | | Reads the snapshot registry: every snapshot, or the file's, as `getfattr -n user.snapshots` lists them | 200, `{snapshots: [{id, name, inode, state, size_bytes, blocks, created_at}], total}` | +| `GET /api/v1/snapshots[?path=/f]` | | Reads the snapshot registry: every snapshot, with `orphan: true` on those whose file is gone, or the file's, as `getfattr -n user.snapshots` lists them | 200, `{snapshots: [{id, name, inode, state, size_bytes, blocks, created_at, orphan}], total}` | | `POST /api/v1/snapshots` | `{"path": "/f", "name": "s1"}` | `setfattr -n user.snapshot -v s1 f` | 201 | | `DELETE /api/v1/snapshots/s1?path=/f` | | `setfattr -n user.snapshot.delete -v s1 f` | 200 | +| `DELETE /api/v1/snapshots/s1` | | Deletes an orphan: a snapshot whose file is gone | 200; 409 if its file exists | | `POST /api/v1/snapshots/s1/clone` | `{"path": "/f", "dest": "/copy"}` | `setfattr -n user.snapshot.clone -v s1:/copy f` | 201 | | `POST /api/v1/clones` | `{"path": "/f", "dest": "/copy"}` | `setfattr -n user.clone -v /copy f` | 201 | | `POST /api/v1/admin/flush-journal` | | Checkpoints the journal now | 200 | diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 3f43fb89..6796ca6a 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -172,6 +172,8 @@ setfattr -n user.clone -v /data.db.copy /mnt/fs/data.db setfattr -x user.snapshot.snap1 /mnt/fs/data.db ``` +A snapshot stays with the file it was taken of. When you delete the file, its snapshots become orphans: `getfattr` on any file no longer lists them, and `GET /api/v1/snapshots` lists them with `orphan: true`. Delete an orphan with `DELETE /api/v1/snapshots/` and no `path`. Snapshots made by older versions are orphans after an upgrade, because they do not record which file they were taken of. `goclusterfs-fsck` warns about each orphan. + Snapshots and clones share blocks with the file and copy a block on its first write. A block is freed when no file, snapshot, or clone uses it. Before you mount an image with snapshots or clones using a newer build, read the upgrade notes in [CHANGELOG.md](../CHANGELOG.md). --- diff --git a/internal/api/writes.go b/internal/api/writes.go index ef660420..42e03d5c 100644 --- a/internal/api/writes.go +++ b/internal/api/writes.go @@ -52,6 +52,8 @@ type SnapshotInfo struct { SizeBytes uint64 `json:"size_bytes"` Blocks uint64 `json:"blocks"` CreatedAt string `json:"created_at,omitempty"` + // Orphan marks a snapshot whose file is gone, or that cannot prove which file it was taken of. + Orphan bool `json:"orphan,omitempty"` } // SnapshotListResponse is returned by GET /api/v1/snapshots. diff --git a/internal/fileops/fileops.go b/internal/fileops/fileops.go index d2bddb0e..fb997134 100644 --- a/internal/fileops/fileops.go +++ b/internal/fileops/fileops.go @@ -20,21 +20,28 @@ import ( // Ops implements api.FileService for the filesystem mounted at a directory. type Ops struct { - root string - registry *snapshot.Registry + root string + snaps Snapshots // checkRoot refuses a root that is not the mounted filesystem. checkRoot func(fd int) error } +// Snapshots is the filesystem's snapshot registry, which knows the file each snapshot was taken of. +type Snapshots interface { + AllSnapshots(ctx context.Context) ([]snapshot.Entry, error) + FileSnapshots(ctx context.Context, inodeNum uint32) ([]*snapshot.Descriptor, error) + DeleteOrphanSnapshot(ctx context.Context, name string) error +} + // New returns Ops for the mount at root, which must already be mounted. -// registry is nil when snapshots are off. -func New(root string, registry *snapshot.Registry) *Ops { +// snaps is nil when snapshots are off. +func New(root string, snaps Snapshots) *Ops { var st unix.Stat_t dev := uint64(0) if unix.Stat(root, &st) == nil { dev = uint64(st.Dev) } - return &Ops{root: root, registry: registry, checkRoot: func(fd int) error { return isOurMount(fd, dev) }} + return &Ops{root: root, snaps: snaps, checkRoot: func(fd int) error { return isOurMount(fd, dev) }} } // isOurMount refuses a root that is no longer this FUSE mount, for example a @@ -64,26 +71,33 @@ func sameDevice(fd int, dev uint64) error { var _ api.FileService = (*Ops)(nil) -// ListSnapshots returns every snapshot, or only those of the file at p. +// ListSnapshots returns every snapshot, marking those whose file is gone, or only those of the file at p. func (o *Ops) ListSnapshots(ctx context.Context, p string) ([]api.SnapshotInfo, error) { - if o.registry == nil { + if o.snaps == nil { return nil, fmt.Errorf("snapshots are %w (-snapshots=false)", api.ErrDisabled) } - var descs []*snapshot.Descriptor - var err error + var entries []snapshot.Entry if p == "" { - descs, err = o.registry.List(ctx) + var err error + if entries, err = o.snaps.AllSnapshots(ctx); err != nil { + return nil, err + } } else { - var st unix.Stat_t - if st, err = o.stat(p); err == nil { - descs, err = o.registry.ListForInode(ctx, uint32(st.Ino)) + st, err := o.stat(p) + if err != nil { + return nil, err + } + descs, err := o.snaps.FileSnapshots(ctx, uint32(st.Ino)) + if err != nil { + return nil, err + } + for _, d := range descs { + entries = append(entries, snapshot.Entry{Descriptor: d}) } } - if err != nil { - return nil, err - } - out := make([]api.SnapshotInfo, 0, len(descs)) - for _, d := range descs { + out := make([]api.SnapshotInfo, 0, len(entries)) + for _, e := range entries { + d := e.Descriptor info := api.SnapshotInfo{ ID: d.SnapshotID, Name: d.GetName(), @@ -91,6 +105,7 @@ func (o *Ops) ListSnapshots(ctx context.Context, p string) ([]api.SnapshotInfo, State: d.State.String(), SizeBytes: d.TotalSize, Blocks: d.TotalBlocks, + Orphan: e.Orphan, } if d.CreateTime > 0 { info.CreatedAt = time.Unix(d.CreateTime, 0).UTC().Format(time.RFC3339) @@ -108,11 +123,18 @@ func (o *Ops) CreateSnapshot(_ context.Context, p, name string) error { return o.setxattr(p, "user.snapshot", name) } -// DeleteSnapshot does what setfattr -n user.snapshot.delete -v name does. -func (o *Ops) DeleteSnapshot(_ context.Context, p, name string) error { +// DeleteSnapshot does what setfattr -n user.snapshot.delete -v name does. Without p it deletes a snapshot whose file +// is gone. +func (o *Ops) DeleteSnapshot(ctx context.Context, p, name string) error { if name == "" || strings.ContainsRune(name, '\x00') { return fmt.Errorf("%w: snapshot name %q", api.ErrInvalidPath, name) } + if p == "" { + if o.snaps == nil { + return fmt.Errorf("snapshots are %w (-snapshots=false)", api.ErrDisabled) + } + return o.snaps.DeleteOrphanSnapshot(ctx, name) + } return o.setxattr(p, "user.snapshot.delete", name) } diff --git a/internal/fsck/fsck.go b/internal/fsck/fsck.go index 7d0a8ffe..0c89fc0d 100644 --- a/internal/fsck/fsck.go +++ b/internal/fsck/fsck.go @@ -390,6 +390,7 @@ type Checker struct { badBitmap map[uint64]bool // bitmap sectors with a bad checksum inodeBlocks map[uint64]int // block -> reference count from inodes inodeUsed map[uint32]bool // inode numbers that are in use + inodeBirth map[uint32]uint64 dirEntryRefs map[uint32]int // inode -> names in directories, not counting "." and ".." dirNames map[uint32]int // directory inode -> names it holds namesIncomplete []string // why dirEntryRefs may miss names @@ -475,6 +476,7 @@ func New(dev *storage.Device, opts Options) *Checker { finished: make(map[Phase]bool), inodeBlocks: make(map[uint64]int), inodeUsed: make(map[uint32]bool), + inodeBirth: make(map[uint32]uint64), dirEntryRefs: make(map[uint32]int), dirNames: make(map[uint32]int), snapshotInodes: make(map[uint32]bool), diff --git a/internal/fsck/phase3_inodes.go b/internal/fsck/phase3_inodes.go index 19d6347d..3a377e7c 100644 --- a/internal/fsck/phase3_inodes.go +++ b/internal/fsck/phase3_inodes.go @@ -67,6 +67,8 @@ type InodeData struct { ExtentDepth uint32 ExtentTreeGen uint64 + BirthGeneration uint64 + // Checksum Checksum uint32 } @@ -160,6 +162,7 @@ func (c *Checker) checkInodes() error { usedInodes++ c.inodeUsed[inodeNum] = true + c.inodeBirth[inodeNum] = inode.BirthGeneration // Validate inode number matches position if inode.InodeNum != inodeNum { @@ -272,6 +275,7 @@ func (c *Checker) parseInode(buf []byte) *InodeData { inode.ExtentCount = binary.LittleEndian.Uint64(buf[176:184]) inode.ExtentDepth = binary.LittleEndian.Uint32(buf[184:188]) inode.ExtentTreeGen = binary.LittleEndian.Uint64(buf[188:196]) + inode.BirthGeneration = binary.LittleEndian.Uint64(buf[196:204]) // Checksum (last 4 bytes) inode.Checksum = binary.LittleEndian.Uint32(buf[252:256]) diff --git a/internal/fsck/phase7_snapshots.go b/internal/fsck/phase7_snapshots.go index d3483532..0058a223 100644 --- a/internal/fsck/phase7_snapshots.go +++ b/internal/fsck/phase7_snapshots.go @@ -26,6 +26,8 @@ const ( // SnapshotFlagHasData marks descriptors whose DataInode holds the captured extents. SnapshotFlagHasData uint8 = 1 << 4 + // SnapshotFlagHasTargetBirth marks descriptors that record the birth generation of the file they were taken of. + SnapshotFlagHasTargetBirth uint8 = 1 << 5 // MaxSnapshotNameLen is the maximum length of a snapshot name. MaxSnapshotNameLen = 48 @@ -53,6 +55,7 @@ type SnapshotDescriptor struct { CreateTime int64 DeleteTime int64 RefCountJobID uint64 + TargetBirth uint64 Name string Checksum uint32 } @@ -161,10 +164,9 @@ func (c *Checker) checkSnapshots() error { seenNames[desc.Name] = desc.SnapshotID } - // Validate target inode - if desc.TargetInode > 0 && !c.inodeUsed[desc.TargetInode] { - c.addProblem(PhaseSnapshots, Error, fmt.Sprintf("snapshot %d", desc.SnapshotID), - fmt.Sprintf("references non-existent inode %d", desc.TargetInode), + if why := c.snapshotOrphaned(desc); why != "" { + c.addProblem(PhaseSnapshots, Warning, fmt.Sprintf("snapshot %d", desc.SnapshotID), + fmt.Sprintf("snapshot %q is an orphan: %s; only the API's full listing shows it, and it can be deleted there", desc.Name, why), false) } @@ -222,6 +224,19 @@ func (c *Checker) checkSnapshots() error { return nil } +// snapshotOrphaned says why desc belongs to no existing file, or returns "". +func (c *Checker) snapshotOrphaned(desc *SnapshotDescriptor) string { + switch { + case !c.inodeUsed[desc.TargetInode]: + return fmt.Sprintf("inode %d, the file it was taken of, is free", desc.TargetInode) + case desc.Flags&SnapshotFlagHasTargetBirth == 0: + return "an older version wrote it without the birth generation that proves which file it was taken of" + case desc.TargetBirth != c.inodeBirth[desc.TargetInode]&(1<<48-1): + return fmt.Sprintf("inode %d now holds a later file than the one it was taken of", desc.TargetInode) + } + return "" +} + // parseSnapshotDescriptor extracts a snapshot descriptor from raw bytes. func parseSnapshotDescriptor(buf []byte) *SnapshotDescriptor { desc := &SnapshotDescriptor{} @@ -232,8 +247,10 @@ func parseSnapshotDescriptor(buf []byte) *SnapshotDescriptor { desc.TargetInode = binary.LittleEndian.Uint32(buf[20:24]) desc.State = buf[24] desc.Flags = buf[25] - // Bytes 26-27 reserved desc.DataInode = binary.LittleEndian.Uint32(buf[28:32]) + if desc.Flags&SnapshotFlagHasTargetBirth != 0 { + desc.TargetBirth = uint64(binary.LittleEndian.Uint16(buf[26:28]))<<32 | uint64(binary.LittleEndian.Uint32(buf[32:36])) + } desc.TotalBlocks = binary.LittleEndian.Uint64(buf[36:44]) desc.TotalSize = binary.LittleEndian.Uint64(buf[44:52]) desc.CreateTime = int64(binary.LittleEndian.Uint64(buf[52:60])) diff --git a/internal/goclusterfs/fs.go b/internal/goclusterfs/fs.go index 10f95557..5866ce7e 100644 --- a/internal/goclusterfs/fs.go +++ b/internal/goclusterfs/fs.go @@ -3363,6 +3363,11 @@ func (f *File) createSnapshot(ctx context.Context, name string) syscall.Errno { log.Printf("[snapshot] Failed to read inode %d: %v", f.inodeNum, err) return syscall.EIO } + target, err := f.fs.snapshotTarget(ctx, src) + if err != nil { + log.Printf("[snapshot] Failed to record the birth generation of inode %d: %v", f.inodeNum, err) + return syscall.EIO + } data, err := f.fs.claimInode(ctx, func(ino *inode.Inode) { ino.Mode, ino.UID, ino.GID = src.Mode, src.UID, src.GID @@ -3374,7 +3379,7 @@ func (f *File) createSnapshot(ctx context.Context, name string) syscall.Errno { return claimErrno(err) } - desc, err := f.fs.snapshotReg.Create(ctx, name, f.inodeNum, 0, + desc, err := f.fs.snapshotReg.CreateFor(ctx, name, target, 0, data.InodeNum, src.AllocatedBlocks, src.Size, snapshot.FlagReadOnly) if err != nil { log.Printf("[snapshot] Failed to add '%s' to the registry: %v", name, err) @@ -3418,8 +3423,8 @@ func (f *File) deleteSnapshot(ctx context.Context, name string) syscall.Errno { return syscall.EIO } - if desc.TargetInode != f.inodeNum { - return syscall.EINVAL + if errno := f.ownsSnapshot(desc); errno != 0 { + return errno } if err := f.fs.dropSnapshot(ctx, desc.SnapshotID); err != nil { @@ -3534,9 +3539,9 @@ func (f *File) createCloneFromSnapshot(ctx context.Context, data string) syscall return syscall.EIO } - if desc.TargetInode != f.inodeNum { - log.Printf("[clone] Snapshot '%s' belongs to inode %d, not %d", snapName, desc.TargetInode, f.inodeNum) - return syscall.EINVAL + if errno := f.ownsSnapshot(desc); errno != 0 { + log.Printf("[clone] Snapshot '%s' was not taken of inode %d", snapName, f.inodeNum) + return errno } if desc.State != snapshot.StateActive { return syscall.EBUSY @@ -3563,7 +3568,7 @@ func (f *File) listSnapshots(ctx context.Context, dest []byte) (uint32, syscall. return 0, syscall.ENOTSUP } - snapshots, err := f.fs.snapshotReg.ListForInode(ctx, f.inodeNum) + snapshots, err := f.fs.FileSnapshots(ctx, f.inodeNum) if err != nil { return 0, syscall.EIO } @@ -3584,6 +3589,93 @@ func (f *File) listSnapshots(ctx context.Context, dest []byte) (uint32, syscall. return uint32(n), 0 } +// ownsSnapshot returns ENOENT unless desc was taken of this file, not of an earlier file in the same inode slot. +func (f *File) ownsSnapshot(desc *snapshot.Descriptor) syscall.Errno { + ino, err := f.fs.inodeTbl.Read(f.inodeNum) + if err != nil { + return syscall.EIO + } + if !desc.BelongsTo(snapshot.Target{Inode: f.inodeNum, Birth: ino.BirthGeneration}) { + return syscall.ENOENT + } + return 0 +} + +// snapshotTarget returns what a snapshot of ino records about its file. An inode older versions wrote gets its birth +// generation now: the generation this write gives it, which no later file in the slot can have. +func (f *FS) snapshotTarget(ctx context.Context, ino *inode.Inode) (snapshot.Target, error) { + if ino.BirthGeneration == 0 { + updated, err := f.updateInode(ctx, ino.InodeNum, func(i *inode.Inode) error { + if i.BirthGeneration == 0 { + i.BirthGeneration = i.MetaGeneration + 1 + } + return nil + }) + if err != nil { + return snapshot.Target{}, err + } + ino = updated + } + return snapshot.Target{Inode: ino.InodeNum, Birth: ino.BirthGeneration}, nil +} + +// FileSnapshots returns the snapshots of the file in inode slot num. +func (f *FS) FileSnapshots(ctx context.Context, num uint32) ([]*snapshot.Descriptor, error) { + ino, err := f.inodeTbl.Read(num) + if err != nil { + return nil, err + } + return f.snapshotReg.ListForTarget(ctx, snapshot.Target{Inode: num, Birth: ino.BirthGeneration}) +} + +// AllSnapshots returns every snapshot, marking those whose file is gone or cannot be proven. +func (f *FS) AllSnapshots(ctx context.Context) ([]snapshot.Entry, error) { + descs, err := f.snapshotReg.List(ctx) + if err != nil { + return nil, err + } + entries := make([]snapshot.Entry, 0, len(descs)) + for _, d := range descs { + orphan, err := f.snapshotOrphan(d) + if err != nil { + return nil, err + } + entries = append(entries, snapshot.Entry{Descriptor: d, Orphan: orphan}) + } + return entries, nil +} + +// DeleteOrphanSnapshot deletes a snapshot whose file is gone. A snapshot of an existing file is deleted through it. +func (f *FS) DeleteOrphanSnapshot(ctx context.Context, name string) error { + desc, err := f.snapshotReg.GetByName(ctx, name) + if errors.Is(err, snapshot.ErrSnapshotNotFound) { + return fmt.Errorf("snapshot %q: %w", name, syscall.ENOENT) + } + if err != nil { + return err + } + orphan, err := f.snapshotOrphan(desc) + if err != nil { + return err + } + if !orphan { + return fmt.Errorf("snapshot %q belongs to inode %d, so delete it through that file: %w", name, desc.TargetInode, syscall.EBUSY) + } + log.Printf("[snapshot] Deleting orphan '%s' (ID=%d) of inode %d", name, desc.SnapshotID, desc.TargetInode) + return f.dropSnapshot(ctx, desc.SnapshotID) +} + +func (f *FS) snapshotOrphan(d *snapshot.Descriptor) (bool, error) { + ino, err := f.inodeTbl.Read(d.TargetInode) + if errors.Is(err, inode.ErrInodeOutOfRange) { + return true, nil + } + if err != nil { + return false, err + } + return ino.Magic == 0 || !d.BelongsTo(snapshot.Target{Inode: d.TargetInode, Birth: ino.BirthGeneration}), nil +} + // blockExtent returns the single block of ext at offset, so per-block checks // such as refcounts look at the right physical block. func blockExtent(ext *extent.Extent, offset uint64) *extent.Extent { diff --git a/internal/goclusterfs/snapshot_owner_test.go b/internal/goclusterfs/snapshot_owner_test.go new file mode 100644 index 00000000..eeddec5b --- /dev/null +++ b/internal/goclusterfs/snapshot_owner_test.go @@ -0,0 +1,134 @@ +package goclusterfs + +import ( + "context" + "strings" + "syscall" + "testing" + + "github.com/supporttools/GoClusterFS/internal/fsck" + "github.com/supporttools/GoClusterFS/internal/snapshot" +) + +func snapshotList(t *testing.T, f *File) string { + t.Helper() + buf := make([]byte, 4096) + n, errno := f.Getxattr(context.Background(), "user.snapshots", buf) + if errno != 0 { + t.Fatalf("inode %d: getxattr user.snapshots: %v", f.inodeNum, errno) + } + return string(buf[:n]) +} + +// snapshotThenReuseSlot snapshots a file as s, deletes the file, and creates a new file in the freed inode slot. +func snapshotThenReuseSlot(t *testing.T) (*FS, *File) { + t.Helper() + ctx := context.Background() + fsys := openFormattedFS(t, newFormattedImage(t)) + old := createFile(t, fsys, "old") + writeAt(t, old, pattern(1, BlockSize), 0) + setxattr(t, old, "user.snapshot", "s") + if got := snapshotList(t, old); !strings.HasPrefix(got, "s:") { + t.Fatalf("the file lists snapshots %q, want s", got) + } + if errno := fsys.Unlink(ctx, "old"); errno != 0 { + t.Fatalf("unlink: %v", errno) + } + reused := createFile(t, fsys, "new") + if reused.inodeNum != old.inodeNum { + t.Fatalf("the new file got inode %d, want the freed inode %d", reused.inodeNum, old.inodeNum) + } + writeAt(t, reused, pattern(2, BlockSize), 0) + return fsys, reused +} + +// A snapshot belongs to the file it was taken of. A later file in the same inode slot must not see, clone or delete it. +func TestSnapshotStaysWithItsFile(t *testing.T) { + ctx := context.Background() + fsys, reused := snapshotThenReuseSlot(t) + + if got := snapshotList(t, reused); got != "" { + t.Errorf("the new file lists %q, the deleted file's snapshots", got) + } + if errno := reused.Setxattr(ctx, "user.snapshot.clone", []byte("s:/leak"), 0); errno != syscall.ENOENT { + t.Errorf("clone of the deleted file's snapshot through the new file: errno %v, want ENOENT", errno) + } + if errno := reused.Setxattr(ctx, "user.snapshot.delete", []byte("s"), 0); errno != syscall.ENOENT { + t.Errorf("delete of the deleted file's snapshot through the new file: errno %v, want ENOENT", errno) + } + if _, err := fsys.snapshotReg.GetByName(ctx, "s"); err != nil { + t.Errorf("snapshot s: %v, want it kept as an orphan", err) + } + expectOrphanReport(t, fsys, `snapshot "s" is an orphan: inode`) +} + +// expectOrphanReport requires fsck to warn about the orphan with want in its message, and to find no error. +func expectOrphanReport(t *testing.T, fsys *FS, want string) { + t.Helper() + r := runFsck(t, fsys, fsck.DefaultOptions()) + found := false + for _, p := range r.Problems { + found = found || p.Severity == fsck.Warning && strings.Contains(p.Message, want) + if p.Severity != fsck.Warning { + t.Errorf("fsck: %s", p) + } + } + if !found { + t.Errorf("fsck does not report the orphan snapshot:%s", problemList(r)) + } +} + +// The full listing marks a deleted file's snapshot as an orphan, and only an orphan can be deleted without its file. +func TestOrphanSnapshotListedAndDeletable(t *testing.T) { + ctx := context.Background() + fsys, reused := snapshotThenReuseSlot(t) + setxattr(t, reused, "user.snapshot", "live") + + all, err := fsys.AllSnapshots(ctx) + if err != nil { + t.Fatal(err) + } + orphan := map[string]bool{} + for _, e := range all { + orphan[e.GetName()] = e.Orphan + } + if len(orphan) != 2 || !orphan["s"] || orphan["live"] { + t.Fatalf("full listing marks orphans %v, want s an orphan and live not", orphan) + } + if err := fsys.DeleteOrphanSnapshot(ctx, "live"); err == nil { + t.Error("deleting the live file's snapshot as an orphan succeeded") + } + if err := fsys.DeleteOrphanSnapshot(ctx, "s"); err != nil { + t.Fatalf("delete orphan s: %v", err) + } + waitSnapshotGone(t, fsys, "s") + if got := snapshotList(t, reused); !strings.HasPrefix(got, "live:") { + t.Errorf("the new file lists %q, want its own snapshot live", got) + } +} + +// Descriptors written before snapshots recorded their file's birth generation cannot prove which file they belong to, +// so they are orphans. +func TestSnapshotWithoutBirthIsOrphan(t *testing.T) { + ctx := context.Background() + fsys := openFormattedFS(t, newFormattedImage(t)) + f := createFile(t, fsys, "f") + desc, err := fsys.snapshotReg.Create(ctx, "legacy", f.inodeNum, 0, 0, 0, 0, snapshot.FlagReadOnly) + if err != nil { + t.Fatal(err) + } + if err := fsys.snapshotReg.Activate(ctx, desc.SnapshotID); err != nil { + t.Fatal(err) + } + if got := snapshotList(t, f); got != "" { + t.Errorf("the file lists %q, a snapshot that cannot prove it belongs to it", got) + } + all, err := fsys.AllSnapshots(ctx) + if err != nil { + t.Fatal(err) + } + if len(all) != 1 || !all[0].Orphan { + t.Errorf("full listing %+v, want legacy as an orphan", all) + } + expectOrphanReport(t, fsys, `snapshot "legacy" is an orphan: an older version wrote it`) +} diff --git a/internal/inode/inode.go b/internal/inode/inode.go index 4be9a261..f78a5770 100644 --- a/internal/inode/inode.go +++ b/internal/inode/inode.go @@ -124,7 +124,8 @@ type Inode struct { ExtentCount uint64 // Total extent count ExtentDepth uint32 // Extent tree depth (0 = inline only) ExtentTreeGen uint64 // Commit generation of the tree root; its low bit picks the root's active half - Reserved [56]byte // Padding to 252 bytes + BirthGeneration uint64 // MetaGeneration when this file took the slot; 0 in inodes older versions wrote + Reserved [48]byte // Padding to 252 bytes Checksum uint32 // CRC32C checksum // PendingTree holds extent-tree changes owned by package extent that are @@ -504,6 +505,7 @@ func (t *Table) newInodeInSlot(num uint32, free *Inode) *Inode { LinkCount: 1, DataGeneration: free.DataGeneration + 1, MetaGeneration: free.MetaGeneration + 1, + BirthGeneration: free.MetaGeneration + 1, LastModifierUUID: t.nodeUUID, } } @@ -838,8 +840,9 @@ func serializeInode(inode *Inode) []byte { binary.LittleEndian.PutUint64(buf[176:184], inode.ExtentCount) binary.LittleEndian.PutUint32(buf[184:188], inode.ExtentDepth) binary.LittleEndian.PutUint64(buf[188:196], inode.ExtentTreeGen) + binary.LittleEndian.PutUint64(buf[196:204], inode.BirthGeneration) - // Reserved bytes 196-251 (56 bytes) - already zero + // Reserved bytes 204-251 (48 bytes) - already zero // Compute checksum (bytes 0-251) checksum := storage.CRC32C(buf[:InodeSize-4]) @@ -889,6 +892,7 @@ func deserializeInode(buf []byte) (*Inode, error) { ExtentCount: binary.LittleEndian.Uint64(buf[176:184]), ExtentDepth: binary.LittleEndian.Uint32(buf[184:188]), ExtentTreeGen: binary.LittleEndian.Uint64(buf[188:196]), + BirthGeneration: binary.LittleEndian.Uint64(buf[196:204]), Checksum: stored, } diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go index 2ccf468f..79fdf3c7 100644 --- a/internal/snapshot/snapshot.go +++ b/internal/snapshot/snapshot.go @@ -113,8 +113,27 @@ const ( // inode that maps the blocks the snapshot captured. Descriptors written by // older versions lack it and hold no data. FlagHasData Flags = 1 << 4 + + // FlagHasTargetBirth marks a descriptor that records its file's birth generation, which tells the file apart from + // a later file in the same inode slot. Without it the snapshot cannot prove which file it belongs to. + FlagHasTargetBirth Flags = 1 << 5 ) +// birthMask keeps the 48 bits of a birth generation a descriptor has room for. +const birthMask = 1<<48 - 1 + +// Target names the file a snapshot was taken of: its inode slot and the birth generation of the file in it. +type Target struct { + Inode uint32 + Birth uint64 +} + +// Entry is a snapshot and whether the file it was taken of is gone. +type Entry struct { + *Descriptor + Orphan bool +} + // Errors var ( ErrSnapshotNotFound = errors.New("snapshot not found") @@ -142,9 +161,9 @@ var ( // 20 4 TargetInode (file/directory being snapshotted) // 24 1 State (Creating/Active/Deleting) // 25 1 Flags (ReadOnly/Writable/Hidden) -// 26 2 Reserved1 +// 26 2 TargetBirth bits 32-47 (only with FlagHasTargetBirth) // 28 4 DataInode (hidden inode with the captured extents; only with FlagHasData) -// 32 4 Reserved2 (older versions wrote an unused extent tree root at 28-35) +// 32 4 TargetBirth bits 0-31 (only with FlagHasTargetBirth; older versions wrote an unused extent tree root at 28-35) // 36 8 TotalBlocks (block count at snapshot time) // 44 8 TotalSize (file size at snapshot time) // 52 8 CreateTime (Unix timestamp) @@ -159,9 +178,8 @@ type Descriptor struct { TargetInode uint32 // Inode number of snapshotted file State State // Current state Flags Flags // Snapshot flags - Reserved1 uint16 // Alignment padding DataInode uint32 // Hidden inode holding the captured extents - Reserved2 uint32 // Reserved + TargetBirth uint64 // Birth generation of the snapshotted file, 48 bits TotalBlocks uint64 // Number of blocks at snapshot time TotalSize uint64 // File size at snapshot time CreateTime int64 // Creation timestamp @@ -181,6 +199,12 @@ func (d *Descriptor) HasData() bool { return d.Flags&FlagHasData != 0 && d.DataInode != 0 } +// BelongsTo reports whether the snapshot was taken of target, the file now in that inode slot. +func (d *Descriptor) BelongsTo(target Target) bool { + return d.Flags&FlagHasTargetBirth != 0 && target.Birth != 0 && + d.TargetInode == target.Inode && d.TargetBirth == target.Birth&birthMask +} + // IsFree returns true if this slot is free (can be reused). func (d *Descriptor) IsFree() bool { return d.Magic == 0 || d.State == StateInvalid @@ -299,6 +323,13 @@ func (r *Registry) unlock() { func (r *Registry) Create(ctx context.Context, name string, targetInode uint32, parentID uint64, dataInode uint32, totalBlocks uint64, totalSize uint64, flags Flags) (*Descriptor, error) { + return r.CreateFor(ctx, name, Target{Inode: targetInode}, parentID, dataInode, totalBlocks, totalSize, flags) +} + +// CreateFor is Create for a target whose birth generation is known, so the snapshot stays with that file. +func (r *Registry) CreateFor(ctx context.Context, name string, target Target, + parentID uint64, dataInode uint32, totalBlocks uint64, + totalSize uint64, flags Flags) (*Descriptor, error) { if len(name) >= MaxSnapshotNameLen { return nil, ErrNameTooLong @@ -327,11 +358,15 @@ func (r *Registry) Create(ctx context.Context, name string, targetInode uint32, if dataInode != 0 { flags |= FlagHasData } + if target.Birth != 0 { + flags |= FlagHasTargetBirth + } desc := &Descriptor{ Magic: DescriptorMagic, SnapshotID: snapshotID, ParentID: parentID, - TargetInode: targetInode, + TargetInode: target.Inode, + TargetBirth: target.Birth & birthMask, State: StateCreating, // Will be marked Active after refcount job Flags: flags | FlagReadOnly, DataInode: dataInode, @@ -415,6 +450,20 @@ func (r *Registry) ListForInode(ctx context.Context, inodeNum uint32) ([]*Descri return snapshots, nil } +// ListForTarget returns the snapshots taken of target. +func (r *Registry) ListForTarget(ctx context.Context, target Target) ([]*Descriptor, error) { + r.mu.RLock() + defer r.mu.RUnlock() + + var snapshots []*Descriptor + for _, desc := range r.descriptors() { + if desc.IsValid() && desc.BelongsTo(target) { + snapshots = append(snapshots, desc) + } + } + return snapshots, nil +} + // GetChildren returns all snapshots whose ParentID matches the given snapshot ID. // Used for merge-up during middle snapshot deletion. func (r *Registry) GetChildren(ctx context.Context, snapshotID uint64) ([]*Descriptor, error) { @@ -844,9 +893,7 @@ func decodeDescriptor(sectorData []byte, descIdx uint64) (*Descriptor, error) { TargetInode: binary.LittleEndian.Uint32(data[20:24]), State: State(data[24]), Flags: Flags(data[25]), - Reserved1: binary.LittleEndian.Uint16(data[26:28]), DataInode: binary.LittleEndian.Uint32(data[28:32]), - Reserved2: binary.LittleEndian.Uint32(data[32:36]), TotalBlocks: binary.LittleEndian.Uint64(data[36:44]), TotalSize: binary.LittleEndian.Uint64(data[44:52]), CreateTime: int64(binary.LittleEndian.Uint64(data[52:60])), @@ -855,6 +902,9 @@ func decodeDescriptor(sectorData []byte, descIdx uint64) (*Descriptor, error) { Checksum: stored, } copy(desc.Name[:], data[76:124]) + if desc.Flags&FlagHasTargetBirth != 0 { + desc.TargetBirth = uint64(binary.LittleEndian.Uint16(data[26:28]))<<32 | uint64(binary.LittleEndian.Uint32(data[32:36])) + } return desc, nil } @@ -880,9 +930,9 @@ func encodeDescriptor(sectorData []byte, descIdx uint64, desc *Descriptor) { binary.LittleEndian.PutUint32(data[20:24], desc.TargetInode) data[24] = byte(desc.State) data[25] = byte(desc.Flags) - binary.LittleEndian.PutUint16(data[26:28], desc.Reserved1) + binary.LittleEndian.PutUint16(data[26:28], uint16(desc.TargetBirth>>32)) binary.LittleEndian.PutUint32(data[28:32], desc.DataInode) - binary.LittleEndian.PutUint32(data[32:36], desc.Reserved2) + binary.LittleEndian.PutUint32(data[32:36], uint32(desc.TargetBirth)) binary.LittleEndian.PutUint64(data[36:44], desc.TotalBlocks) binary.LittleEndian.PutUint64(data[44:52], desc.TotalSize) binary.LittleEndian.PutUint64(data[52:60], uint64(desc.CreateTime)) diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go index 1c0dffa9..507219f2 100644 --- a/internal/snapshot/snapshot_test.go +++ b/internal/snapshot/snapshot_test.go @@ -1171,3 +1171,34 @@ func TestRegistryCreateRefusesTakenName(t *testing.T) { t.Fatalf("reusing a removed name: %v", err) } } + +// A descriptor keeps its file's birth generation on disk, and matches only that file. +func TestDescriptorKeepsTargetBirth(t *testing.T) { + dev, cleanup := createTestDevice(t, 10<<20) + defer cleanup() + ctx := context.Background() + + target := Target{Inode: 7, Birth: 1<<40 + 5} + if _, err := NewRegistry(dev, 0, 4*SectorSize, nil).CreateFor(ctx, "s", target, InvalidSnapshotID, 0, 1, 1024, FlagReadOnly); err != nil { + t.Fatal(err) + } + reg := NewRegistry(dev, 0, 4*SectorSize, nil) + got, err := reg.GetByName(ctx, "s") + if err != nil { + t.Fatal(err) + } + if !got.BelongsTo(target) { + t.Errorf("snapshot of %+v read back with birth %d, want it to belong to its file", target, got.TargetBirth) + } + for _, other := range []Target{{7, target.Birth + 1}, {8, target.Birth}, {7, 0}} { + if got.BelongsTo(other) { + t.Errorf("snapshot of %+v belongs to %+v", target, other) + } + if list, err := reg.ListForTarget(ctx, other); err != nil || len(list) != 0 { + t.Errorf("ListForTarget(%+v) = %d snapshots, %v; want none", other, len(list), err) + } + } + if list, err := reg.ListForTarget(ctx, target); err != nil || len(list) != 1 { + t.Errorf("ListForTarget(%+v) = %d snapshots, %v; want 1", target, len(list), err) + } +} diff --git a/tests/test-manifest.yaml b/tests/test-manifest.yaml index 66abdf66..5b1cef5c 100644 --- a/tests/test-manifest.yaml +++ b/tests/test-manifest.yaml @@ -1500,6 +1500,34 @@ scenarios: environments: go: internal/journal/record_order_test.go::TestRecoverReplaysVersion1Journal + - id: snapshot-stays-with-its-file + name: A snapshot stays with the file it was taken of + description: A file is snapshotted and deleted, and a new file takes its inode slot; the new file lists, clones and deletes none of the old file's snapshots, which fsck reports as orphans + priority: go_only + environments: + go: internal/goclusterfs/snapshot_owner_test.go::TestSnapshotStaysWithItsFile + + - id: orphan-snapshot-listed-and-deletable + name: Orphan snapshots are listed and deleted without their file + description: The full listing marks a deleted file's snapshot as an orphan and the new file's as not; only the orphan can be deleted without a file + priority: go_only + environments: + go: internal/goclusterfs/snapshot_owner_test.go::TestOrphanSnapshotListedAndDeletable + + - id: snapshot-without-birth-is-orphan + name: A snapshot without its file's birth generation is an orphan + description: A descriptor as older versions wrote it cannot prove which file it belongs to; the file does not list it, the full listing marks it an orphan, and fsck reports it + priority: go_only + environments: + go: internal/goclusterfs/snapshot_owner_test.go::TestSnapshotWithoutBirthIsOrphan + + - id: snapshot-descriptor-target-birth + name: A snapshot descriptor keeps its file's birth generation + description: A descriptor read back from disk belongs to its file and to no other inode number or birth generation + priority: go_only + environments: + go: internal/snapshot/snapshot_test.go::TestDescriptorKeepsTargetBirth + - id: fsck-repair-in-use name: fsck -repair refuses an in-use filesystem description: -repair refuses while a node heartbeats or a stopped node's journal is dirty; -force overrides only the journal check; dry runs and checks still run From db7eacc86dd456db436aca069b71f2d765e06cf6 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Fri, 13 Nov 2026 23:16:28 -0600 Subject: [PATCH 2/2] test: treat older snapshots as orphans in tests, and allow orphan warnings in VM fsck checks Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 3 +-- internal/goclusterfs/snapshot_test.go | 9 +++++---- tests/vm/scripts/run-vm-tests.sh | 9 +++++---- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3fe834f..12df3793 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,8 +18,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), - Directory entries now fill all 4 KB chunks of a directory block. When an older binary removes the first entry of a later chunk, it grows the last entry of the chunk before it across the boundary. This version then stops listing that entry, and fsck reports the directory block. - Run `goclusterfs-fsck -repair` before the first mount after upgrading an image that has snapshots or clones. Unmount the device on every node first. - Older versions marked a file's blocks shared, on snapshot or clone, without counting the owners. Until `goclusterfs-fsck -repair` counts them, this version never frees such blocks, so deleting those files leaks their space, and it refuses to snapshot or clone them (`ENOTSUP`). In-place writes to an old clone still change its source. -- Snapshots made by older versions captured no data. They still list and delete, but cloning one fails with `ENOTSUP`. -- Snapshots now record the birth generation of the file they were taken of, so a later file that reuses the inode number cannot list, clone or delete them. Snapshots older versions made do not record it, so after the upgrade they are orphans: `getfattr -n user.snapshots` no longer lists them, `GET /api/v1/snapshots` lists them with `orphan: true`, and `DELETE /api/v1/snapshots/` without `path` deletes them. Older binaries keep the birth generation in inode bytes 196–203 only while they do not rewrite the inode, so upgrade all nodes together. +- Snapshots now record the birth generation of the file they were taken of, so a later file that reuses the inode number cannot list, clone or delete them. Snapshots older versions made do not record it, and some captured no data, so after the upgrade they are orphans: `getfattr -n user.snapshots` no longer lists them, `GET /api/v1/snapshots` lists them with `orphan: true`, and `DELETE /api/v1/snapshots/` without `path` deletes them. Older binaries keep the birth generation in inode bytes 196–203 only while they do not rewrite the inode, so upgrade all nodes together. - `-skip-recovery` now takes the slot number from the recovery error, as `-skip-recovery=N`, and discards a journal only when the mount takes slot N. A bare `-skip-recovery` is a usage error. ### Added diff --git a/internal/goclusterfs/snapshot_test.go b/internal/goclusterfs/snapshot_test.go index e3d0dfd8..6f33cb4b 100644 --- a/internal/goclusterfs/snapshot_test.go +++ b/internal/goclusterfs/snapshot_test.go @@ -817,11 +817,12 @@ func TestOldSnapshotAndCloneNeverFreeSharedBlocks(t *testing.T) { writeAt(t, clone, pattern(6, 100), 10) stillAllocated("writing the old clone") - if errno := src.Setxattr(ctx, "user.snapshot.clone", []byte("old-snap:/x"), 0); errno != syscall.ENOTSUP { - t.Fatalf("clone of a snapshot without data: errno %v, want ENOTSUP", errno) + // An older version's snapshot does not record its file, so it is an orphan. + if errno := src.Setxattr(ctx, "user.snapshot.clone", []byte("old-snap:/x"), 0); errno != syscall.ENOENT { + t.Fatalf("clone of an orphan snapshot through the file: errno %v, want ENOENT", errno) } - if errno := src.Removexattr(ctx, "user.snapshot.old-snap"); errno != 0 { - t.Fatalf("delete old snapshot: %v", errno) + if err := fsys.DeleteOrphanSnapshot(ctx, "old-snap"); err != nil { + t.Fatalf("delete old snapshot: %v", err) } waitSnapshotGone(t, fsys, "old-snap") stillAllocated("deleting the old snapshot") diff --git a/tests/vm/scripts/run-vm-tests.sh b/tests/vm/scripts/run-vm-tests.sh index 00f904ad..e0ff120d 100755 --- a/tests/vm/scripts/run-vm-tests.sh +++ b/tests/vm/scripts/run-vm-tests.sh @@ -3300,8 +3300,9 @@ restart_and_expect_only_leaks() { fi done + # Snapshots of files earlier tests deleted are orphans, which is not damage. out=$(ssh_run 1 "$fsck /dev/vda 2>&1") - problems=$(echo "$out" | grep -E '^ *\[(WARN|ERROR)\]') + problems=$(echo "$out" | grep -E '^ *\[(WARN|ERROR)\]' | grep -v 'is an orphan') if echo "$problems" | grep -v -e '^$' -e 'leaked' -e 'will never be freed' -e 'unnamed: no directory names it' -e 'belongs to no snapshot' | grep -q .; then log_error "fsck after the power failure found more than leaks:" echo "$problems" | head -20 >&2 @@ -3310,10 +3311,10 @@ restart_and_expect_only_leaks() { if [[ -n "$problems" ]]; then log_info "fsck found $(echo "$problems" | wc -l) leaks; repairing" ssh_run 1 "$fsck -repair /dev/vda > /dev/null 2>&1" - out=$(ssh_run 1 "$fsck /dev/vda 2>&1") - if echo "$out" | grep -qE '^ *\[(WARN|ERROR)\]'; then + out=$(ssh_run 1 "$fsck /dev/vda 2>&1" | grep -E '^ *\[(WARN|ERROR)\]' | grep -v 'is an orphan') + if [[ -n "$out" ]]; then log_error "fsck after repair still finds problems:" - echo "$out" | grep -E '^ *\[(WARN|ERROR)\]' | head -20 >&2 + echo "$out" | head -20 >&2 return 1 fi fi