Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions internal/lock/fairness_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,10 @@ func TestHolderYieldsOnlyToLiveWaiters(t *testing.T) {
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
defer func(d time.Duration) { holdTurn = d }(holdTurn)
holdTurn = 50 * time.Millisecond
dev, cleanup := createTestDevice(t, 1<<20)
defer cleanup()
holder := newTestNode(dev, 0, 1<<20, 1)
holder.timing.holdTurn = 50 * time.Millisecond
if tc.view {
reg := newFakeRegistry()
reg.set(5, tc.state)
Expand All @@ -98,7 +97,7 @@ func TestHolderYieldsOnlyToLiveWaiters(t *testing.T) {
stop := make(chan struct{})
var wg sync.WaitGroup
wg.Go(func() { reclaimAfterExpiry(t, holder, LockTypeInode, 3, stop) })
time.Sleep(4 * holdTurn)
time.Sleep(4 * holder.timing.holdTurn)
close(stop)
wg.Wait()

Expand Down
15 changes: 3 additions & 12 deletions internal/lock/holds.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,15 +9,6 @@ import (
"time"
)

// longHold is how long a node holds a record before it warns that the hold may
// have leaked. Under an owner view a claim never goes stale, so a hold that is
// never released lasts until unmount.
var longHold = time.Minute

// releaseRetry is how often the watcher clears again a slot whose release write
// failed. Other nodes wait for the record until it is cleared.
var releaseRetry = time.Second

// Start runs a watcher that logs a warning for each record this node has held
// for more than longHold. Stop ends it.
func (m *Manager) Start() {
Expand All @@ -44,9 +35,9 @@ func (m *Manager) Stop() {

func (m *Manager) watchHolds(stop <-chan struct{}, done chan<- struct{}) {
defer close(done)
tick := time.NewTicker(longHold / 6)
tick := time.NewTicker(m.timing.longHold / 6)
defer tick.Stop()
retry := time.NewTicker(releaseRetry)
retry := time.NewTicker(m.timing.releaseRetry)
defer retry.Stop()
for {
select {
Expand All @@ -66,7 +57,7 @@ func (m *Manager) warnLongHolds() {
m.mu.Lock()
var warnings []string
for recordID, rec := range m.records {
if rec.keys == 0 || rec.warnedLong || age(rec.since) <= longHold {
if rec.keys == 0 || rec.warnedLong || age(rec.since) <= m.timing.longHold {
continue
}
rec.warnedLong = true
Expand Down
6 changes: 2 additions & 4 deletions internal/lock/holds_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,19 +11,17 @@ import (
)

func TestLongHoldIsLoggedOnce(t *testing.T) {
old := longHold
t.Cleanup(func() { longHold = old })
longHold = 120 * time.Millisecond
dev, cleanup := createTestDevice(t, 1<<20)
defer cleanup()
holder := newTestNode(dev, 0, 1<<20, 1)
holder.timing.longHold = 120 * time.Millisecond

var logs bytes.Buffer
log.SetOutput(&logs)
t.Cleanup(func() { log.SetOutput(os.Stderr) })
holder.Start()
requireAcquired(t, holder, LockTypeInode, 2, LockModeExclusive)
time.Sleep(5 * longHold)
time.Sleep(5 * holder.timing.longHold)
holder.Stop()
log.SetOutput(os.Stderr)

Expand Down
69 changes: 44 additions & 25 deletions internal/lock/lock.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ type Manager struct {
onAcquire OnAcquireFunc

waiting atomic.Int64
timing timings

slotWrites, rowReads, conflicts, waits, deferred atomic.Uint64
// claimTimes is a histogram of the time from a claim's slot write to the end of its row read.
Expand Down Expand Up @@ -250,22 +251,39 @@ func covers(held, want LockMode) bool {
return held == LockModeExclusive || held == want
}

// renewAfter is the age of a slot after which this node writes it again before
// relying on it, so other nodes never see a slot in use as stale.
var renewAfter = StaleThreshold / 3

// acquireTimeout is how long Acquire waits at most; tests shorten it.
var acquireTimeout = DefaultTimeout

// tornExpiry is how long a slot must keep failing its checksum before other
// nodes ignore it. A live node rewrites its slot within renewAfter.
var tornExpiry = StaleThreshold

var (
cacheGrace = CacheGrace
holdTurn = HoldTurn
yieldGap = YieldGap
)
// timings are a manager's timing settings. Tests shorten them on the manager
// they test, before using it, so no test changes what another test's timers read.
type timings struct {
cacheGrace, holdTurn, yieldGap time.Duration
// renewAfter is the age of a slot after which this node writes it again
// before relying on it, so other nodes never see a slot in use as stale.
renewAfter time.Duration
// acquireTimeout is how long Acquire waits at most.
acquireTimeout time.Duration
// tornExpiry is how long a slot must keep failing its checksum before other
// nodes ignore it. A live node rewrites its slot within renewAfter.
tornExpiry time.Duration
// longHold is how long a node holds a record before it warns that the hold
// may have leaked. Under an owner view a claim never goes stale, so a hold
// that is never released lasts until unmount.
longHold time.Duration
// releaseRetry is how often the watcher clears again a slot whose release
// write failed. Other nodes wait for the record until it is cleared.
releaseRetry time.Duration
}

func defaultTimings() timings {
return timings{
cacheGrace: CacheGrace,
holdTurn: HoldTurn,
yieldGap: YieldGap,
renewAfter: StaleThreshold / 3,
acquireTimeout: DefaultTimeout,
tornExpiry: StaleThreshold,
longHold: time.Minute,
releaseRetry: time.Second,
}
}

// releaseParallelism is how many kept records ReleaseCached frees at once.
const releaseParallelism = 8
Expand Down Expand Up @@ -301,6 +319,7 @@ func NewManager(dev *storage.Device, offset, size uint64, nodeUUID [16]byte, nod
adopted: make(map[uint64]adoptedSlot),
unreleased: make(map[uint64]bool),
tornSince: make(map[slotRef]tornSighting),
timing: defaultTimings(),
}
}

Expand Down Expand Up @@ -533,7 +552,7 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin
return m.giveUp(ErrLockTimeout, lockType, resourceID, mode, startTime, fmt.Sprintf(
"the journal replay of node %x (slot %d), which holds record %d, failed or was abandoned", by.occupant[:8], by.slot, recordID))
}
if by.failing() && time.Since(lastFailingLog) >= acquireTimeout {
if by.failing() && time.Since(lastFailingLog) >= m.timing.acquireTimeout {
lastFailingLog = time.Now()
log.Printf("[WARN] lock: type=%d res=%d mode=%d has waited %v for node %x (slot %d), which holds it and is failing or being recovered; waiting until its journal replay ends",
lockType, resourceID, mode, time.Since(startTime).Round(time.Millisecond), by.occupant[:8], by.slot)
Expand All @@ -542,7 +561,7 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin
// Check context cancellation, but allow at least 3 retries. A wait gives
// up after acquireTimeout of waiting for live holders, or for failing ones
// while this goroutine holds other locks; a replay's wait never does.
if retryCount >= 3 && (ctx.Err() != nil || !replay && liveWait > acquireTimeout) {
if retryCount >= 3 && (ctx.Err() != nil || !replay && liveWait > m.timing.acquireTimeout) {
if failingWhileNested {
return m.giveUp(ErrReplayBlocked, lockType, resourceID, mode, startTime, fmt.Sprintf(
"this goroutine holds other locks, and record %d is held by node %x (slot %d), which is failing or being recovered and whose replay may need one of them",
Expand Down Expand Up @@ -581,7 +600,7 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi
defer gate.Unlock()

m.mu.Lock()
if rec, ok := m.records[recordID]; ok && rec.keys == 0 && (!covers(rec.mode, mode) || age(rec.renewed) > renewAfter) {
if rec, ok := m.records[recordID]; ok && rec.keys == 0 && (!covers(rec.mode, mode) || age(rec.renewed) > m.timing.renewAfter) {
// A cached hold too weak for this request, or old enough that another node
// may soon ignore it as stale, is claimed afresh over this node's slot.
delete(m.records, recordID)
Expand All @@ -596,7 +615,7 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi
return false, false, blocker{}, nil
}
if m.turnOverLocked(recordID, rec) && !m.goroutineHoldsRecordLocked(recordID, gid) &&
(waited < holdTurn || !m.goroutineHoldsAnyLocked(gid)) {
(waited < m.timing.holdTurn || !m.goroutineHoldsAnyLocked(gid)) {
if rec.keys > 0 {
m.mu.Unlock()
return false, false, blocker{}, nil
Expand All @@ -605,7 +624,7 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi
m.mu.Unlock()
return false, false, blocker{}, m.releaseRecord(recordID)
}
renew := age(rec.renewed) > renewAfter
renew := age(rec.renewed) > m.timing.renewAfter
m.mu.Unlock()
if renew {
if err := m.renewRecord(recordID, key, rec); err != nil {
Expand Down Expand Up @@ -665,7 +684,7 @@ func (m *Manager) giveUp(err error, lockType LockType, resourceID uint64, mode L
// saw another node waiting.
func (m *Manager) turnOverLocked(recordID uint64, rec *heldRecord) bool {
seen, waited := m.contested[recordID]
return age(rec.since) > holdTurn || waited && age(seen) > holdTurn
return age(rec.since) > m.timing.holdTurn || waited && age(seen) > m.timing.holdTurn
}

// endTurnLocked drops this node's hold on recordID, which the caller releases
Expand All @@ -674,7 +693,7 @@ func (m *Manager) turnOverLocked(recordID uint64, rec *heldRecord) bool {
func (m *Manager) endTurnLocked(recordID uint64) {
delete(m.records, recordID)
delete(m.contested, recordID)
m.yieldUntil[recordID] = time.Now().Add(yieldGap)
m.yieldUntil[recordID] = time.Now().Add(m.timing.yieldGap)
}

func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) bool {
Expand All @@ -683,7 +702,7 @@ func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) boo
return false
}
rec := m.records[held.RecordID]
if !covers(rec.mode, mode) || age(rec.renewed) > renewAfter {
if !covers(rec.mode, mode) || age(rec.renewed) > m.timing.renewAfter {
return false
}
if m.turnOverLocked(held.RecordID, rec) && !m.goroutineHoldsRecordLocked(held.RecordID, gid) {
Expand Down Expand Up @@ -795,7 +814,7 @@ func (m *Manager) dropRecordKey(recordID uint64) error {
gen := rec.cacheGen
m.mu.Unlock()

time.AfterFunc(cacheGrace, func() { m.expireCached(recordID, gen) })
time.AfterFunc(m.timing.cacheGrace, func() { m.expireCached(recordID, gen) })
return nil
}

Expand Down
29 changes: 14 additions & 15 deletions internal/lock/owner_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,10 @@ func (f *fakeRegistry) view(slot int, fresh bool) ([16]byte, OwnerState) {
return [16]byte{}, OwnerReleased
}

func shortenAcquireTimeout(t *testing.T, d time.Duration) {
t.Helper()
old := acquireTimeout
t.Cleanup(func() { acquireTimeout = old })
acquireTimeout = d
// shortenAcquireTimeout sets d as the acquire timeout of ms and returns it.
func shortenAcquireTimeout(d time.Duration, ms ...*Manager) time.Duration {
setTimings(func(t *timings) { t.acquireTimeout = d }, ms...)
return d
}

// acquireInBackground starts an Acquire with no deadline and returns its result.
Expand Down Expand Up @@ -96,29 +95,29 @@ func TestFailingHoldersOldClaimBlocksUntilRegistrySlotEmpties(t *testing.T) {
}

func TestWaitForFailingHolderOutlastsTimeout(t *testing.T) {
shortenAcquireTimeout(t, 300*time.Millisecond)
dev, cleanup := createTestDevice(t, 1<<20)
defer cleanup()
holder := newTestNode(dev, 0, 1<<20, 1)
waiter := newTestNode(dev, 0, 1<<20, 2)
timeout := shortenAcquireTimeout(300*time.Millisecond, holder, waiter)
reg := newFakeRegistry()
reg.set(1, OwnerFailing)
waiter.SetOwnerView(reg.view)

requireAcquired(t, holder, LockTypeInode, 2, LockModeExclusive)
result := acquireInBackground(waiter, LockTypeInode, 2)
requireWaiting(t, result, 3*acquireTimeout)
requireWaiting(t, result, 3*timeout)

reg.set(1, OwnerReleased)
requireResult(t, result, nil)
}

func TestWaitForLiveHolderStillTimesOut(t *testing.T) {
shortenAcquireTimeout(t, 300*time.Millisecond)
dev, cleanup := createTestDevice(t, 1<<20)
defer cleanup()
holder := newTestNode(dev, 0, 1<<20, 1)
waiter := newTestNode(dev, 0, 1<<20, 2)
shortenAcquireTimeout(300*time.Millisecond, holder, waiter)
reg := newFakeRegistry()
reg.set(1, OwnerAlive)
waiter.SetOwnerView(reg.view)
Expand Down Expand Up @@ -179,12 +178,12 @@ func TestTornSlotFreesWhenItsRegistrySlotEmpties(t *testing.T) {
// while it replays that journal at mount. Peers wait for the replay, however
// long, though the node itself is alive.
func TestPeersWaitOutLiveNodesReplayOfItsPredecessor(t *testing.T) {
shortenAcquireTimeout(t, 300*time.Millisecond)
dev, cleanup := createTestDevice(t, reclaimRegion)
defer cleanup()
predecessor := NewManager(dev, 0, reclaimRegion, [16]byte{1}, 1)
successor := NewManager(dev, 0, reclaimRegion, [16]byte{5}, 1)
waiter := NewManager(dev, 0, reclaimRegion, [16]byte{2}, 2)
timeout := shortenAcquireTimeout(300*time.Millisecond, predecessor, successor, waiter)
reg := newFakeRegistry()
reg.set(1, OwnerAlive)
waiter.SetOwnerView(reg.view)
Expand All @@ -195,7 +194,7 @@ func TestPeersWaitOutLiveNodesReplayOfItsPredecessor(t *testing.T) {
t.Fatalf("AdoptOwner = %v, %v; want the predecessor's record", adopted, err)
}
result := acquireInBackground(waiter, LockTypeInode, 2)
requireWaiting(t, result, 3*acquireTimeout)
requireWaiting(t, result, 3*timeout)

if err := successor.ReleaseAdopted(adopted); err != nil {
t.Fatal(err)
Expand Down Expand Up @@ -247,12 +246,12 @@ func TestReplaysSharingRecordTakeTurns(t *testing.T) {
// gives up after the timeout and lets Y go, and the replay, which waits without
// a limit, takes Y and can finish.
func TestNestedWaitOnReplayingNodeYieldsToReplay(t *testing.T) {
shortenAcquireTimeout(t, 300*time.Millisecond)
dev, cleanup := createTestDevice(t, reclaimRegion)
defer cleanup()
dead := NewManager(dev, 0, reclaimRegion, [16]byte{1}, 1)
replayer := NewManager(dev, 0, reclaimRegion, [16]byte{3}, 3)
peer := NewManager(dev, 0, reclaimRegion, [16]byte{4}, 4)
timeout := shortenAcquireTimeout(300*time.Millisecond, dead, replayer, peer)
reg := newFakeRegistry()
reg.set(1, OwnerFailing)
reg.set(3, OwnerAlive)
Expand Down Expand Up @@ -294,13 +293,13 @@ func TestNestedWaitOnReplayingNodeYieldsToReplay(t *testing.T) {
go func() {
replayResult <- replayer.Acquire(ForReplay(context.Background()), LockTypeInode, inodeY, LockModeExclusive)
}()
time.Sleep(acquireTimeout / 2)
time.Sleep(timeout / 2)
close(askX)

requireResult(t, peerResult, ErrLockTimeout)
requireResult(t, replayResult, nil)
if waited := time.Since(replayStart); waited <= acquireTimeout {
t.Errorf("replay waited %v, want longer than the %v timeout", waited, acquireTimeout)
if waited := time.Since(replayStart); waited <= timeout {
t.Errorf("replay waited %v, want longer than the %v timeout", waited, timeout)
}
log.SetOutput(os.Stderr)
if want := fmt.Sprintf("held by node %x (slot 1)", [8]byte{1}); !strings.Contains(logs.String(), want) {
Expand Down Expand Up @@ -345,11 +344,11 @@ func TestReplayDoesNotPassOverLiveNodesReplayClaims(t *testing.T) {
// A caller that can release its locks and retry gets ErrReplayBlocked at once,
// instead of holding them for the timeout while a replay may need one.
func TestRetryableNestedWaitOnFailingHolderReturnsAtOnce(t *testing.T) {
shortenAcquireTimeout(t, 5*time.Second)
dev, cleanup := createTestDevice(t, reclaimRegion)
defer cleanup()
dead := NewManager(dev, 0, reclaimRegion, [16]byte{1}, 1)
peer := NewManager(dev, 0, reclaimRegion, [16]byte{4}, 4)
shortenAcquireTimeout(5*time.Second, dead, peer)
reg := newFakeRegistry()
reg.set(1, OwnerFailing)
reg.set(4, OwnerAlive)
Expand Down
2 changes: 1 addition & 1 deletion internal/lock/release_retry_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ func TestFailedReleaseWriteIsRetried(t *testing.T) {
failNext.Store(true)
release(t, a, LockTypeInode, 3)

if err := acquireWithin(b, 5*releaseRetry, LockTypeInode, 3, LockModeExclusive); err != nil {
if err := acquireWithin(b, 5*a.timing.releaseRetry, LockTypeInode, 3, LockModeExclusive); err != nil {
t.Fatalf("the other node is still kept out after one failed release write: %v", err)
}
release(t, b, LockTypeInode, 3)
Expand Down
Loading
Loading