Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 21 additions & 1 deletion cmd/mount/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,7 @@ func main() {
log.Fatalf("Failed to register node: %v", err)
}
nodeReg.SetSlotLostHandler(func(slot int, owner [16]byte) {
log.Fatalf("Node registry slot %d now belongs to node %x; stopping so two nodes never share a journal", slot, owner[:8])
log.Fatalf("Node registry slot %d is no longer this node's (owner %x); stopping so two nodes never share a journal", slot, owner[:8])
})
if err := nodeReg.StartHeartbeat(); err != nil {
log.Fatalf("Failed to start node heartbeat: %v", err)
Expand All @@ -134,6 +134,11 @@ func main() {

// Create lock manager for distributed locking
lockMgr := lock.NewManager(dev, lockOffset, lockSize, nodeUUID, slotIdx)
nodeReg.SetOwnerDeadHandler(func(owner [16]byte) func() { return adoptLocks(lockMgr, owner) })
releasePrevious := func() {}
if owner, ok := nodeReg.DeadPreviousOwner(); ok {
releasePrevious = adoptLocks(lockMgr, owner)
}

// Create managers with distributed locking enabled
// NOTE: Managers must be created BEFORE journal recovery so we can replay transactions
Expand Down Expand Up @@ -169,6 +174,7 @@ func main() {
} else if *skipRecovery {
log.Println("Journal: WARNING - skipping recovery (emergency mode)")
}
releasePrevious()

// Create journal for write-ahead logging (crash consistency)
var j *journal.Journal
Expand Down Expand Up @@ -412,3 +418,17 @@ func main() {

log.Println("Filesystem unmounted.")
}

// adoptLocks takes over a dead node's exclusive locks before its journal is
// replayed and returns the function that frees them after the replay.
func adoptLocks(locks *lock.Manager, owner [16]byte) func() {
adopted, err := locks.AdoptOwner(owner)
if err != nil {
log.Printf("Warning: Failed to take over the locks of dead node %x: %v", owner[:8], err)
}
return func() {
if err := locks.ReleaseAdopted(adopted); err != nil {
log.Printf("Warning: Failed to release the locks taken over from dead node %x: %v", owner[:8], err)
}
}
}
1 change: 1 addition & 0 deletions docs/OPERATIONS.md
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ rm /mnt/fs/.snapshots/snapshot_name

- **Clocks:** Nodes judge each other's heartbeats by wall clock. A node more than about 10 seconds ahead treats live peers as failed, replays their journals and takes their registry slots. Run NTP on every node.
- **Restarts:** A restarted mount takes back the registry slot of a mount that exited on the same host (same hostname and kernel boot ID) and replays its journal before serving. In the same PID namespace it checks the old process ID; across PID namespaces it waits for the old slot's heartbeat to go stale.
- **Crashed node's locks:** Once a node is proven dead, the node replaying its journal takes over its exclusive locks and frees them after the replay, so no other node uses them mid-replay. Proven dead means its mount process is gone (same host and PID namespace), or its heartbeat is more than 20 seconds old and has not changed for 15 seconds by the observing node's own clock. A node paused for more than about 20 seconds therefore counts as dead; it exits on its next heartbeat. A dead node's shared locks, and the locks of a node not proven dead, are freed once they are 15 seconds old.
- **Node count:** The device has 64 registry slots, each with its own 4 MB journal. A crashed node's slot frees once another node replays its journal, or when the same host mounts again.

### Known Gaps in Slot Ownership
Expand Down
115 changes: 115 additions & 0 deletions internal/lock/reclaim.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
package lock

import (
"log"
"time"
)

// adoptSettle is how long AdoptOwner waits before re-reading the records it
// took, so a peer's racing stale takeover of one of them is noticed.
const adoptSettle = 50 * time.Millisecond

// AdoptOwner makes this node the holder of every exclusive lock record owner
// holds and returns their IDs. Call it only once owner is proven dead, before
// replaying owner's journal, and pass the result to ReleaseAdopted after the
// replay: until then peers cannot take the locks of the dead node's unfinished
// operations, while this node's own acquires reuse them.
//
// Shared records are left to the staleness rule: OwnerUUID names only the
// latest reader, who may have released already while others still read.
func (m *Manager) AdoptOwner(owner [16]byte) ([]uint64, error) {
var taken []uint64
for recordID := uint64(0); recordID < m.size/RecordSize; recordID++ {
ok, err := m.adoptRecord(recordID, owner)
if err != nil {
return taken, err
}
if ok {
taken = append(taken, recordID)
}
}
if len(taken) == 0 {
return nil, nil
}

time.Sleep(adoptSettle)
var adopted []uint64
for _, recordID := range taken {
if m.holdsExclusive(recordID) {
adopted = append(adopted, recordID)
}
}
log.Printf("lock: took over %d lock records of dead node %x", len(adopted), owner[:8])
return adopted, nil
}

func (m *Manager) adoptRecord(recordID uint64, owner [16]byte) (bool, error) {
gate := m.recordGate(recordID)
gate.Lock()
defer gate.Unlock()

for {
record, err := m.readRecord(recordID)
if err != nil {
return false, err
}
if record.Mode != LockModeExclusive || record.OwnerUUID != owner {
return false, nil
}
adopted := *record
adopted.OwnerUUID = m.nodeUUID
adopted.OwnerSlot = uint32(m.nodeSlot)
adopted.AcquireTime = time.Now().Unix()
adopted.Sequence++
ok, err := m.casRecord(recordID, record, &adopted)
if err != nil {
return false, err
}
if ok {
return true, nil
}
}
}

// ReleaseAdopted frees the records AdoptOwner returned that this node still
// holds without using them itself; a record in use is freed by its own release.
func (m *Manager) ReleaseAdopted(records []uint64) error {
for _, recordID := range records {
if err := m.releaseAdopted(recordID); err != nil {
return err
}
}
return nil
}

func (m *Manager) releaseAdopted(recordID uint64) error {
gate := m.recordGate(recordID)
gate.Lock()
defer gate.Unlock()

m.mu.Lock()
_, inUse := m.records[recordID]
m.mu.Unlock()
if inUse {
return nil
}

for {
record, err := m.readRecord(recordID)
if err != nil {
return err
}
if record.Mode != LockModeExclusive || record.OwnerUUID != m.nodeUUID {
return nil
}
ok, err := m.casRecord(recordID, record, &Record{Sequence: record.Sequence + 1})
if err != nil || ok {
return err
}
}
}

func (m *Manager) holdsExclusive(recordID uint64) bool {
record, err := m.readRecord(recordID)
return err == nil && record.Mode == LockModeExclusive && record.OwnerUUID == m.nodeUUID
}
100 changes: 100 additions & 0 deletions internal/lock/reclaim_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
package lock

import (
"context"
"testing"
"time"
)

func distinctRecords(t *testing.T, m *Manager, resources ...uint64) {
t.Helper()
records := map[uint64]bool{}
for _, res := range resources {
records[m.hashToRecord(LockTypeInode, res)] = true
}
if len(records) != len(resources) {
t.Fatal("test resources share a lock record; pick others")
}
}

func TestAdoptOwnerHoldsDeadNodesLocksUntilReleased(t *testing.T) {
dev, cleanup := createTestDevice(t, 1<<20)
defer cleanup()

dead := NewManager(dev, 0, 1<<20, [16]byte{1}, 1)
alive := NewManager(dev, 0, 1<<20, [16]byte{2}, 2)
replayer := NewManager(dev, 0, 1<<20, [16]byte{3}, 3)
peer := NewManager(dev, 0, 1<<20, [16]byte{4}, 4)
distinctRecords(t, replayer, 2, 5, 7)
ctx := context.Background()

for _, res := range []uint64{2, 5} {
if err := dead.Acquire(ctx, LockTypeInode, res, LockModeExclusive); err != nil {
t.Fatal(err)
}
}
if err := alive.Acquire(ctx, LockTypeInode, 7, LockModeExclusive); err != nil {
t.Fatal(err)
}

adopted, err := replayer.AdoptOwner([16]byte{1})
if err != nil {
t.Fatalf("AdoptOwner: %v", err)
}
if len(adopted) != 2 {
t.Fatalf("adopted %d records, want the dead node's 2 exclusive records", len(adopted))
}
if err := acquireWithin(peer, time.Second, LockTypeInode, 2, LockModeExclusive); err == nil {
t.Error("a peer took a dead node's lock while its journal was being replayed")
}
if err := acquireWithin(replayer, time.Second, LockTypeInode, 2, LockModeExclusive); err != nil {
t.Errorf("the replaying node could not use a lock it adopted: %v", err)
}
if err := replayer.Release(LockTypeInode, 2); err != nil {
t.Fatal(err)
}

if err := replayer.ReleaseAdopted(adopted); err != nil {
t.Fatalf("ReleaseAdopted: %v", err)
}
for _, res := range []uint64{2, 5} {
if err := acquireWithin(peer, 2*time.Second, LockTypeInode, res, LockModeExclusive); err != nil {
t.Errorf("lock %d still held after the replay released it: %v", res, err)
}
}
held, err := replayer.readRecord(replayer.hashToRecord(LockTypeInode, 7))
if err != nil {
t.Fatal(err)
}
if held.Mode != LockModeExclusive || held.OwnerUUID != ([16]byte{2}) {
t.Errorf("live node's lock changed: mode %d owner %x", held.Mode, held.OwnerUUID)
}
}

func TestAdoptOwnerLeavesSharedRecordOfReleasedReader(t *testing.T) {
dev, cleanup := createTestDevice(t, 1<<20)
defer cleanup()

reader := NewManager(dev, 0, 1<<20, [16]byte{2}, 2)
crashed := NewManager(dev, 0, 1<<20, [16]byte{1}, 1)
replayer := NewManager(dev, 0, 1<<20, [16]byte{3}, 3)
writer := NewManager(dev, 0, 1<<20, [16]byte{4}, 4)
ctx := context.Background()

if err := reader.Acquire(ctx, LockTypeInode, 9, LockModeShared); err != nil {
t.Fatal(err)
}
if err := crashed.Acquire(ctx, LockTypeInode, 9, LockModeShared); err != nil {
t.Fatal(err)
}
if err := crashed.Release(LockTypeInode, 9); err != nil {
t.Fatal(err)
}

if adopted, err := replayer.AdoptOwner([16]byte{1}); err != nil || len(adopted) != 0 {
t.Fatalf("AdoptOwner = %v, %v; want nothing taken from a reader that already released", adopted, err)
}
if err := acquireWithin(writer, time.Second, LockTypeInode, 9, LockModeExclusive); err == nil {
t.Error("a writer took a record another node is still reading")
}
}
45 changes: 45 additions & 0 deletions internal/node/liveness.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,19 @@ import (
// recoveryLease is how long a peer may hold a dead slot's recovery before others take it over.
const recoveryLease = 60 * time.Second

// deathMargin is added to HeartbeatTimeout before this node's own clock counts
// an unchanged heartbeat as proof that its node is dead rather than slow.
const deathMargin = HeartbeatTimeout / 2

// sighting is what failure detection last saw in a slot, timed by this node's monotonic clock.
type sighting struct {
uuid [16]byte
seq uint64
beat int64
since time.Time
changedAt time.Time
}

var processStart = time.Now().UnixNano()

func readBootID() (id [16]byte) {
Expand Down Expand Up @@ -64,3 +77,35 @@ func (r *Registry) failed(slot *Slot, timeout time.Duration) bool {
}
return false
}

// observe records a slot's heartbeat as failure detection sees it. Called with r.mu held.
func (r *Registry) observe(slotIdx int, slot *Slot) {
last, seen := r.sightings[slotIdx]
if seen && last.uuid == slot.UUID && last.seq == slot.HeartbeatSeq && last.beat == slot.HeartbeatTime {
return
}
now := time.Now()
next := sighting{uuid: slot.UUID, seq: slot.HeartbeatSeq, beat: slot.HeartbeatTime, since: now}
if seen && last.uuid == slot.UUID {
next.changedAt = now
}
r.sightings[slotIdx] = next
}

// recentlyAlive reports whether this node saw the slot's heartbeat advance within
// HeartbeatTimeout, whatever the heartbeat's wall-clock time says. Called with r.mu held.
func (r *Registry) recentlyAlive(slotIdx int) bool {
last, seen := r.sightings[slotIdx]
return seen && !last.changedAt.IsZero() && time.Since(last.changedAt) < HeartbeatTimeout
}

// provenDead reports whether a slot's node is dead, not merely slow: its mount
// process is gone, or this node has seen the same heartbeat for longer than
// HeartbeatTimeout plus deathMargin. Called with r.mu held.
func (r *Registry) provenDead(slotIdx int, slot *Slot) bool {
if r.deadLocalOwner(slot) {
return true
}
last, seen := r.sightings[slotIdx]
return seen && last.uuid == slot.UUID && time.Since(last.since) >= HeartbeatTimeout+deathMargin
}
Loading
Loading