From 087e6896abe790a8fdf656d3ba02f844e6e132b9 Mon Sep 17 00:00:00 2001 From: Matthew Mattox Date: Mon, 16 Nov 2026 11:04:33 -0600 Subject: [PATCH] test(lock): give each manager its own timings, so tests stop writing package globals Lock tests shortened holdTurn, cacheGrace, renewAfter, acquireTimeout, tornExpiry and longHold by writing package variables and restoring them in a defer, while cache-expiry timers and watchers started by earlier tests still read them. The nightly race run caught one: a test's deferred holdTurn restore against another test's expireCached. These settings and yieldGap and releaseRetry are now fields of the manager, set to the defaults in NewManager. Tests set them on the managers they create, before using them. Co-Authored-By: Claude Opus 5.5 --- internal/lock/fairness_test.go | 5 +-- internal/lock/holds.go | 15 ++----- internal/lock/holds_test.go | 6 +-- internal/lock/lock.go | 69 ++++++++++++++++++----------- internal/lock/owner_test.go | 29 ++++++------ internal/lock/release_retry_test.go | 2 +- internal/lock/safety_test.go | 57 +++++++++++------------- internal/lock/slots.go | 8 ++-- 8 files changed, 96 insertions(+), 95 deletions(-) diff --git a/internal/lock/fairness_test.go b/internal/lock/fairness_test.go index e0b7038c..df8c9263 100644 --- a/internal/lock/fairness_test.go +++ b/internal/lock/fairness_test.go @@ -80,11 +80,10 @@ func TestHolderYieldsOnlyToLiveWaiters(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - defer func(d time.Duration) { holdTurn = d }(holdTurn) - holdTurn = 50 * time.Millisecond dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() holder := newTestNode(dev, 0, 1<<20, 1) + holder.timing.holdTurn = 50 * time.Millisecond if tc.view { reg := newFakeRegistry() reg.set(5, tc.state) @@ -98,7 +97,7 @@ func TestHolderYieldsOnlyToLiveWaiters(t *testing.T) { stop := make(chan struct{}) var wg sync.WaitGroup wg.Go(func() { reclaimAfterExpiry(t, holder, LockTypeInode, 3, stop) }) - time.Sleep(4 * holdTurn) + time.Sleep(4 * holder.timing.holdTurn) close(stop) wg.Wait() diff --git a/internal/lock/holds.go b/internal/lock/holds.go index acf882fc..cae63238 100644 --- a/internal/lock/holds.go +++ b/internal/lock/holds.go @@ -9,15 +9,6 @@ import ( "time" ) -// longHold is how long a node holds a record before it warns that the hold may -// have leaked. Under an owner view a claim never goes stale, so a hold that is -// never released lasts until unmount. -var longHold = time.Minute - -// releaseRetry is how often the watcher clears again a slot whose release write -// failed. Other nodes wait for the record until it is cleared. -var releaseRetry = time.Second - // Start runs a watcher that logs a warning for each record this node has held // for more than longHold. Stop ends it. func (m *Manager) Start() { @@ -44,9 +35,9 @@ func (m *Manager) Stop() { func (m *Manager) watchHolds(stop <-chan struct{}, done chan<- struct{}) { defer close(done) - tick := time.NewTicker(longHold / 6) + tick := time.NewTicker(m.timing.longHold / 6) defer tick.Stop() - retry := time.NewTicker(releaseRetry) + retry := time.NewTicker(m.timing.releaseRetry) defer retry.Stop() for { select { @@ -66,7 +57,7 @@ func (m *Manager) warnLongHolds() { m.mu.Lock() var warnings []string for recordID, rec := range m.records { - if rec.keys == 0 || rec.warnedLong || age(rec.since) <= longHold { + if rec.keys == 0 || rec.warnedLong || age(rec.since) <= m.timing.longHold { continue } rec.warnedLong = true diff --git a/internal/lock/holds_test.go b/internal/lock/holds_test.go index 0f5aa7b9..7e0c217c 100644 --- a/internal/lock/holds_test.go +++ b/internal/lock/holds_test.go @@ -11,19 +11,17 @@ import ( ) func TestLongHoldIsLoggedOnce(t *testing.T) { - old := longHold - t.Cleanup(func() { longHold = old }) - longHold = 120 * time.Millisecond dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() holder := newTestNode(dev, 0, 1<<20, 1) + holder.timing.longHold = 120 * time.Millisecond var logs bytes.Buffer log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(os.Stderr) }) holder.Start() requireAcquired(t, holder, LockTypeInode, 2, LockModeExclusive) - time.Sleep(5 * longHold) + time.Sleep(5 * holder.timing.longHold) holder.Stop() log.SetOutput(os.Stderr) diff --git a/internal/lock/lock.go b/internal/lock/lock.go index f0d158de..c896bbbb 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -170,6 +170,7 @@ type Manager struct { onAcquire OnAcquireFunc waiting atomic.Int64 + timing timings slotWrites, rowReads, conflicts, waits, deferred atomic.Uint64 // claimTimes is a histogram of the time from a claim's slot write to the end of its row read. @@ -250,22 +251,39 @@ func covers(held, want LockMode) bool { return held == LockModeExclusive || held == want } -// renewAfter is the age of a slot after which this node writes it again before -// relying on it, so other nodes never see a slot in use as stale. -var renewAfter = StaleThreshold / 3 - -// acquireTimeout is how long Acquire waits at most; tests shorten it. -var acquireTimeout = DefaultTimeout - -// tornExpiry is how long a slot must keep failing its checksum before other -// nodes ignore it. A live node rewrites its slot within renewAfter. -var tornExpiry = StaleThreshold - -var ( - cacheGrace = CacheGrace - holdTurn = HoldTurn - yieldGap = YieldGap -) +// timings are a manager's timing settings. Tests shorten them on the manager +// they test, before using it, so no test changes what another test's timers read. +type timings struct { + cacheGrace, holdTurn, yieldGap time.Duration + // renewAfter is the age of a slot after which this node writes it again + // before relying on it, so other nodes never see a slot in use as stale. + renewAfter time.Duration + // acquireTimeout is how long Acquire waits at most. + acquireTimeout time.Duration + // tornExpiry is how long a slot must keep failing its checksum before other + // nodes ignore it. A live node rewrites its slot within renewAfter. + tornExpiry time.Duration + // longHold is how long a node holds a record before it warns that the hold + // may have leaked. Under an owner view a claim never goes stale, so a hold + // that is never released lasts until unmount. + longHold time.Duration + // releaseRetry is how often the watcher clears again a slot whose release + // write failed. Other nodes wait for the record until it is cleared. + releaseRetry time.Duration +} + +func defaultTimings() timings { + return timings{ + cacheGrace: CacheGrace, + holdTurn: HoldTurn, + yieldGap: YieldGap, + renewAfter: StaleThreshold / 3, + acquireTimeout: DefaultTimeout, + tornExpiry: StaleThreshold, + longHold: time.Minute, + releaseRetry: time.Second, + } +} // releaseParallelism is how many kept records ReleaseCached frees at once. const releaseParallelism = 8 @@ -301,6 +319,7 @@ func NewManager(dev *storage.Device, offset, size uint64, nodeUUID [16]byte, nod adopted: make(map[uint64]adoptedSlot), unreleased: make(map[uint64]bool), tornSince: make(map[slotRef]tornSighting), + timing: defaultTimings(), } } @@ -533,7 +552,7 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin return m.giveUp(ErrLockTimeout, lockType, resourceID, mode, startTime, fmt.Sprintf( "the journal replay of node %x (slot %d), which holds record %d, failed or was abandoned", by.occupant[:8], by.slot, recordID)) } - if by.failing() && time.Since(lastFailingLog) >= acquireTimeout { + if by.failing() && time.Since(lastFailingLog) >= m.timing.acquireTimeout { lastFailingLog = time.Now() log.Printf("[WARN] lock: type=%d res=%d mode=%d has waited %v for node %x (slot %d), which holds it and is failing or being recovered; waiting until its journal replay ends", lockType, resourceID, mode, time.Since(startTime).Round(time.Millisecond), by.occupant[:8], by.slot) @@ -542,7 +561,7 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin // Check context cancellation, but allow at least 3 retries. A wait gives // up after acquireTimeout of waiting for live holders, or for failing ones // while this goroutine holds other locks; a replay's wait never does. - if retryCount >= 3 && (ctx.Err() != nil || !replay && liveWait > acquireTimeout) { + if retryCount >= 3 && (ctx.Err() != nil || !replay && liveWait > m.timing.acquireTimeout) { if failingWhileNested { return m.giveUp(ErrReplayBlocked, lockType, resourceID, mode, startTime, fmt.Sprintf( "this goroutine holds other locks, and record %d is held by node %x (slot %d), which is failing or being recovered and whose replay may need one of them", @@ -581,7 +600,7 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi defer gate.Unlock() m.mu.Lock() - if rec, ok := m.records[recordID]; ok && rec.keys == 0 && (!covers(rec.mode, mode) || age(rec.renewed) > renewAfter) { + if rec, ok := m.records[recordID]; ok && rec.keys == 0 && (!covers(rec.mode, mode) || age(rec.renewed) > m.timing.renewAfter) { // A cached hold too weak for this request, or old enough that another node // may soon ignore it as stale, is claimed afresh over this node's slot. delete(m.records, recordID) @@ -596,7 +615,7 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi return false, false, blocker{}, nil } if m.turnOverLocked(recordID, rec) && !m.goroutineHoldsRecordLocked(recordID, gid) && - (waited < holdTurn || !m.goroutineHoldsAnyLocked(gid)) { + (waited < m.timing.holdTurn || !m.goroutineHoldsAnyLocked(gid)) { if rec.keys > 0 { m.mu.Unlock() return false, false, blocker{}, nil @@ -605,7 +624,7 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi m.mu.Unlock() return false, false, blocker{}, m.releaseRecord(recordID) } - renew := age(rec.renewed) > renewAfter + renew := age(rec.renewed) > m.timing.renewAfter m.mu.Unlock() if renew { if err := m.renewRecord(recordID, key, rec); err != nil { @@ -665,7 +684,7 @@ func (m *Manager) giveUp(err error, lockType LockType, resourceID uint64, mode L // saw another node waiting. func (m *Manager) turnOverLocked(recordID uint64, rec *heldRecord) bool { seen, waited := m.contested[recordID] - return age(rec.since) > holdTurn || waited && age(seen) > holdTurn + return age(rec.since) > m.timing.holdTurn || waited && age(seen) > m.timing.holdTurn } // endTurnLocked drops this node's hold on recordID, which the caller releases @@ -674,7 +693,7 @@ func (m *Manager) turnOverLocked(recordID uint64, rec *heldRecord) bool { func (m *Manager) endTurnLocked(recordID uint64) { delete(m.records, recordID) delete(m.contested, recordID) - m.yieldUntil[recordID] = time.Now().Add(yieldGap) + m.yieldUntil[recordID] = time.Now().Add(m.timing.yieldGap) } func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) bool { @@ -683,7 +702,7 @@ func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) boo return false } rec := m.records[held.RecordID] - if !covers(rec.mode, mode) || age(rec.renewed) > renewAfter { + if !covers(rec.mode, mode) || age(rec.renewed) > m.timing.renewAfter { return false } if m.turnOverLocked(held.RecordID, rec) && !m.goroutineHoldsRecordLocked(held.RecordID, gid) { @@ -795,7 +814,7 @@ func (m *Manager) dropRecordKey(recordID uint64) error { gen := rec.cacheGen m.mu.Unlock() - time.AfterFunc(cacheGrace, func() { m.expireCached(recordID, gen) }) + time.AfterFunc(m.timing.cacheGrace, func() { m.expireCached(recordID, gen) }) return nil } diff --git a/internal/lock/owner_test.go b/internal/lock/owner_test.go index 5d3e57c5..9c1702c3 100644 --- a/internal/lock/owner_test.go +++ b/internal/lock/owner_test.go @@ -43,11 +43,10 @@ func (f *fakeRegistry) view(slot int, fresh bool) ([16]byte, OwnerState) { return [16]byte{}, OwnerReleased } -func shortenAcquireTimeout(t *testing.T, d time.Duration) { - t.Helper() - old := acquireTimeout - t.Cleanup(func() { acquireTimeout = old }) - acquireTimeout = d +// shortenAcquireTimeout sets d as the acquire timeout of ms and returns it. +func shortenAcquireTimeout(d time.Duration, ms ...*Manager) time.Duration { + setTimings(func(t *timings) { t.acquireTimeout = d }, ms...) + return d } // acquireInBackground starts an Acquire with no deadline and returns its result. @@ -96,29 +95,29 @@ func TestFailingHoldersOldClaimBlocksUntilRegistrySlotEmpties(t *testing.T) { } func TestWaitForFailingHolderOutlastsTimeout(t *testing.T) { - shortenAcquireTimeout(t, 300*time.Millisecond) dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() holder := newTestNode(dev, 0, 1<<20, 1) waiter := newTestNode(dev, 0, 1<<20, 2) + timeout := shortenAcquireTimeout(300*time.Millisecond, holder, waiter) reg := newFakeRegistry() reg.set(1, OwnerFailing) waiter.SetOwnerView(reg.view) requireAcquired(t, holder, LockTypeInode, 2, LockModeExclusive) result := acquireInBackground(waiter, LockTypeInode, 2) - requireWaiting(t, result, 3*acquireTimeout) + requireWaiting(t, result, 3*timeout) reg.set(1, OwnerReleased) requireResult(t, result, nil) } func TestWaitForLiveHolderStillTimesOut(t *testing.T) { - shortenAcquireTimeout(t, 300*time.Millisecond) dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() holder := newTestNode(dev, 0, 1<<20, 1) waiter := newTestNode(dev, 0, 1<<20, 2) + shortenAcquireTimeout(300*time.Millisecond, holder, waiter) reg := newFakeRegistry() reg.set(1, OwnerAlive) waiter.SetOwnerView(reg.view) @@ -179,12 +178,12 @@ func TestTornSlotFreesWhenItsRegistrySlotEmpties(t *testing.T) { // while it replays that journal at mount. Peers wait for the replay, however // long, though the node itself is alive. func TestPeersWaitOutLiveNodesReplayOfItsPredecessor(t *testing.T) { - shortenAcquireTimeout(t, 300*time.Millisecond) dev, cleanup := createTestDevice(t, reclaimRegion) defer cleanup() predecessor := NewManager(dev, 0, reclaimRegion, [16]byte{1}, 1) successor := NewManager(dev, 0, reclaimRegion, [16]byte{5}, 1) waiter := NewManager(dev, 0, reclaimRegion, [16]byte{2}, 2) + timeout := shortenAcquireTimeout(300*time.Millisecond, predecessor, successor, waiter) reg := newFakeRegistry() reg.set(1, OwnerAlive) waiter.SetOwnerView(reg.view) @@ -195,7 +194,7 @@ func TestPeersWaitOutLiveNodesReplayOfItsPredecessor(t *testing.T) { t.Fatalf("AdoptOwner = %v, %v; want the predecessor's record", adopted, err) } result := acquireInBackground(waiter, LockTypeInode, 2) - requireWaiting(t, result, 3*acquireTimeout) + requireWaiting(t, result, 3*timeout) if err := successor.ReleaseAdopted(adopted); err != nil { t.Fatal(err) @@ -247,12 +246,12 @@ func TestReplaysSharingRecordTakeTurns(t *testing.T) { // gives up after the timeout and lets Y go, and the replay, which waits without // a limit, takes Y and can finish. func TestNestedWaitOnReplayingNodeYieldsToReplay(t *testing.T) { - shortenAcquireTimeout(t, 300*time.Millisecond) dev, cleanup := createTestDevice(t, reclaimRegion) defer cleanup() dead := NewManager(dev, 0, reclaimRegion, [16]byte{1}, 1) replayer := NewManager(dev, 0, reclaimRegion, [16]byte{3}, 3) peer := NewManager(dev, 0, reclaimRegion, [16]byte{4}, 4) + timeout := shortenAcquireTimeout(300*time.Millisecond, dead, replayer, peer) reg := newFakeRegistry() reg.set(1, OwnerFailing) reg.set(3, OwnerAlive) @@ -294,13 +293,13 @@ func TestNestedWaitOnReplayingNodeYieldsToReplay(t *testing.T) { go func() { replayResult <- replayer.Acquire(ForReplay(context.Background()), LockTypeInode, inodeY, LockModeExclusive) }() - time.Sleep(acquireTimeout / 2) + time.Sleep(timeout / 2) close(askX) requireResult(t, peerResult, ErrLockTimeout) requireResult(t, replayResult, nil) - if waited := time.Since(replayStart); waited <= acquireTimeout { - t.Errorf("replay waited %v, want longer than the %v timeout", waited, acquireTimeout) + if waited := time.Since(replayStart); waited <= timeout { + t.Errorf("replay waited %v, want longer than the %v timeout", waited, timeout) } log.SetOutput(os.Stderr) if want := fmt.Sprintf("held by node %x (slot 1)", [8]byte{1}); !strings.Contains(logs.String(), want) { @@ -345,11 +344,11 @@ func TestReplayDoesNotPassOverLiveNodesReplayClaims(t *testing.T) { // A caller that can release its locks and retry gets ErrReplayBlocked at once, // instead of holding them for the timeout while a replay may need one. func TestRetryableNestedWaitOnFailingHolderReturnsAtOnce(t *testing.T) { - shortenAcquireTimeout(t, 5*time.Second) dev, cleanup := createTestDevice(t, reclaimRegion) defer cleanup() dead := NewManager(dev, 0, reclaimRegion, [16]byte{1}, 1) peer := NewManager(dev, 0, reclaimRegion, [16]byte{4}, 4) + shortenAcquireTimeout(5*time.Second, dead, peer) reg := newFakeRegistry() reg.set(1, OwnerFailing) reg.set(4, OwnerAlive) diff --git a/internal/lock/release_retry_test.go b/internal/lock/release_retry_test.go index a1e97e43..85940ada 100644 --- a/internal/lock/release_retry_test.go +++ b/internal/lock/release_retry_test.go @@ -31,7 +31,7 @@ func TestFailedReleaseWriteIsRetried(t *testing.T) { failNext.Store(true) release(t, a, LockTypeInode, 3) - if err := acquireWithin(b, 5*releaseRetry, LockTypeInode, 3, LockModeExclusive); err != nil { + if err := acquireWithin(b, 5*a.timing.releaseRetry, LockTypeInode, 3, LockModeExclusive); err != nil { t.Fatalf("the other node is still kept out after one failed release write: %v", err) } release(t, b, LockTypeInode, 3) diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go index 87d9af84..a241d84b 100644 --- a/internal/lock/safety_test.go +++ b/internal/lock/safety_test.go @@ -23,6 +23,13 @@ func newTestNode(dev *storage.Device, offset, size uint64, n int) *Manager { return NewManager(dev, offset, size, uuid, n) } +// setTimings changes the timings of ms. Call it before ms are used. +func setTimings(set func(*timings), ms ...*Manager) { + for _, m := range ms { + set(&m.timing) + } +} + func acquireWithin(m *Manager, d time.Duration, lt LockType, id uint64, mode LockMode) error { ctx, cancel := context.WithTimeout(context.Background(), d) defer cancel() @@ -142,8 +149,7 @@ func TestConcurrentAcquireOnOneNodeKeepsOtherNodesOut(t *testing.T) { a := newTestNode(dev, 0, 1<<20, 1) b := newTestNode(dev, 0, 1<<20, 2) // The workers wait for each other while holding, so all must join the hold, however late the scheduler starts them. - defer func(d time.Duration) { holdTurn = d }(holdTurn) - holdTurn = time.Minute + setTimings(func(t *timings) { t.holdTurn = time.Minute }, a, b) const workers = 4 for iter := 0; iter < 30; iter++ { @@ -274,8 +280,7 @@ func TestJoiningKeyRenewsBusyRecord(t *testing.T) { defer cleanup() a := newTestNode(dev, 0, RowSize, 1) b := newTestNode(dev, 0, RowSize, 2) - defer func(d time.Duration) { renewAfter = d }(renewAfter) - renewAfter = 0 + setTimings(func(t *timings) { t.renewAfter = 0 }, a, b) requireAcquired(t, a, LockTypeInode, 1, LockModeExclusive) ageSlot(t, a, LockTypeInode, 1, pastStale) @@ -314,7 +319,7 @@ func TestPausedHolderReleaseKeepsReclaimedLock(t *testing.T) { requireAcquired(t, b, LockTypeInode, 3, LockModeExclusive) release(t, a, LockTypeInode, 3) - time.Sleep(cacheGrace + 50*time.Millisecond) + time.Sleep(a.timing.cacheGrace + 50*time.Millisecond) requireBlocked(t, c, LockTypeInode, 3, LockModeExclusive) } @@ -327,8 +332,6 @@ func TestLockRegionStaysBoundedAcrossCycles(t *testing.T) { ) dev, cleanup := createTestDevice(t, guard+regionSize+guard) defer cleanup() - defer func(g time.Duration) { cacheGrace = g }(cacheGrace) - cacheGrace = time.Millisecond sentinel := storage.NewAlignedBuffer(guard) for i := range sentinel.Bytes() { @@ -344,6 +347,7 @@ func TestLockRegionStaysBoundedAcrossCycles(t *testing.T) { newTestNode(dev, guard, regionSize, 1), newTestNode(dev, guard, regionSize, 2), } + setTimings(func(t *timings) { t.cacheGrace = time.Millisecond }, nodes...) var wg sync.WaitGroup errs := make(chan error, len(nodes)) for n, m := range nodes { @@ -396,8 +400,7 @@ func TestReleasedHoldIsCachedForGrace(t *testing.T) { defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) b := newTestNode(dev, 0, 1<<20, 2) - defer func(g time.Duration) { cacheGrace = g }(cacheGrace) - cacheGrace = 200 * time.Millisecond + setTimings(func(t *timings) { t.cacheGrace = 200 * time.Millisecond }, a, b) requireAcquired(t, a, LockTypeInode, 6, LockModeExclusive) if err := a.Release(LockTypeInode, 6); err != nil { @@ -407,8 +410,8 @@ func TestReleasedHoldIsCachedForGrace(t *testing.T) { start := time.Now() requireAcquired(t, b, LockTypeInode, 6, LockModeExclusive) - if waited := time.Since(start); waited > cacheGrace+4*RetryMaxDelay { - t.Fatalf("other node waited %v for a cached hold, want at most about the %v grace", waited, cacheGrace) + if waited := time.Since(start); waited > a.timing.cacheGrace+4*RetryMaxDelay { + t.Fatalf("other node waited %v for a cached hold, want at most about the %v grace", waited, a.timing.cacheGrace) } } @@ -424,8 +427,8 @@ func TestIdleCachedHoldFreesBeforeTurnEnds(t *testing.T) { } start := time.Now() requireAcquired(t, b, LockTypeInode, 7, LockModeExclusive) - if waited := time.Since(start); waited >= holdTurn { - t.Fatalf("other node waited %v for an idle hold, want less than the %v turn a busy holder gets", waited, holdTurn) + if waited := time.Since(start); waited >= a.timing.holdTurn { + t.Fatalf("other node waited %v for an idle hold, want less than the %v turn a busy holder gets", waited, a.timing.holdTurn) } } @@ -434,9 +437,7 @@ func TestPausedNodeDropsCachedHoldTakenOver(t *testing.T) { defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) b := newTestNode(dev, 0, 1<<20, 2) - defer func(d time.Duration) { renewAfter = d }(renewAfter) - defer func(g time.Duration) { cacheGrace = g }(cacheGrace) - cacheGrace = time.Minute + setTimings(func(t *timings) { t.cacheGrace = time.Minute }, a, b) requireAcquired(t, a, LockTypeInode, 8, LockModeExclusive) if err := a.Release(LockTypeInode, 8); err != nil { @@ -445,7 +446,7 @@ func TestPausedNodeDropsCachedHoldTakenOver(t *testing.T) { // A pauses past StaleThreshold with the write hold cached, and B takes the record over. ageSlot(t, a, LockTypeInode, 8, pastStale) requireAcquired(t, b, LockTypeInode, 8, LockModeExclusive) - renewAfter = 0 + a.timing.renewAfter = 0 requireBlocked(t, a, LockTypeInode, 8, LockModeExclusive) } @@ -455,8 +456,7 @@ func TestReleaseCachedFreesHoldsAtOnce(t *testing.T) { defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) b := newTestNode(dev, 0, 1<<20, 2) - defer func(g time.Duration) { cacheGrace = g }(cacheGrace) - cacheGrace = time.Minute + setTimings(func(t *timings) { t.cacheGrace = time.Minute }, a, b) requireAcquired(t, a, LockTypeInode, 9, LockModeExclusive) if err := a.Release(LockTypeInode, 9); err != nil { @@ -575,8 +575,7 @@ func TestReleaseCachedWithSlowWritesRunsInParallel(t *testing.T) { dev, cleanup := createTestDevice(t, region) defer cleanup() a := newTestNode(dev, 0, region, 1) - defer func(g time.Duration) { cacheGrace = g }(cacheGrace) - cacheGrace = time.Minute + a.timing.cacheGrace = time.Minute records := map[uint64]bool{} for id := uint64(100); len(records) < 16; id++ { @@ -622,8 +621,7 @@ func TestAcquireWithoutDeadlineGivesUp(t *testing.T) { defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) b := newTestNode(dev, 0, 1<<20, 2) - defer func(d time.Duration) { acquireTimeout = d }(acquireTimeout) - acquireTimeout = 200 * time.Millisecond + b.timing.acquireTimeout = 200 * time.Millisecond requireAcquired(t, a, LockTypeInode, 40, LockModeExclusive) errc := make(chan error, 1) @@ -768,13 +766,12 @@ func TestTornSlotCountsAsClaim(t *testing.T) { func TestTornSlotExpires(t *testing.T) { dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() - defer func(d time.Duration) { tornExpiry = d }(tornExpiry) - tornExpiry = 200 * time.Millisecond a := newTestNode(dev, 0, 1<<20, 1) + a.timing.tornExpiry = 200 * time.Millisecond tearSlot(t, a, LockTypeInode, 51, 2) requireBlocked(t, a, LockTypeInode, 51, LockModeExclusive) - time.Sleep(tornExpiry) + time.Sleep(a.timing.tornExpiry) requireAcquired(t, a, LockTypeInode, 51, LockModeExclusive) } @@ -784,13 +781,12 @@ func TestTornSlotExpires(t *testing.T) { func TestTornSlotWaitRestartsWhenBytesChange(t *testing.T) { dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() - defer func(d time.Duration) { tornExpiry = d }(tornExpiry) - tornExpiry = 200 * time.Millisecond a := newTestNode(dev, 0, 1<<20, 1) + a.timing.tornExpiry = 200 * time.Millisecond tearSlot(t, a, LockTypeInode, 51, 2) requireBlocked(t, a, LockTypeInode, 51, LockModeExclusive) - time.Sleep(tornExpiry) + time.Sleep(a.timing.tornExpiry) tearSlot(t, a, LockTypeInode, 51, 2) requireBlocked(t, a, LockTypeInode, 51, LockModeExclusive) } @@ -968,8 +964,7 @@ func TestCachedHoldRejoinNeedsNoDiskIO(t *testing.T) { dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) - defer func(g time.Duration) { cacheGrace = g }(cacheGrace) - cacheGrace = time.Minute + a.timing.cacheGrace = time.Minute requireAcquired(t, a, LockTypeInode, 7, LockModeExclusive) if err := a.Release(LockTypeInode, 7); err != nil { t.Fatalf("Release: %v", err) diff --git a/internal/lock/slots.go b/internal/lock/slots.go index ce3a3b6e..14db8520 100644 --- a/internal/lock/slots.go +++ b/internal/lock/slots.go @@ -265,7 +265,7 @@ func (m *Manager) tornTooLong(ref slotRef, torn []byte) bool { m.tornSince[ref] = tornSighting{since: time.Now(), sum: sum} return false } - return age(seen.since) > tornExpiry + return age(seen.since) > m.timing.tornExpiry } // ownAdoptedLocked reports whether AdoptOwner kept recordID in this node's own @@ -335,7 +335,7 @@ func (m *Manager) dropClaimWaiter(recordID uint64) { func (m *Manager) claim(recordID uint64, key lockKey, mode LockMode) (written time.Time, won bool, by blocker, err error) { m.mu.Lock() c := m.claims[recordID] - rewrite := c.written.IsZero() || c.mode != mode || age(c.written) > renewAfter + rewrite := c.written.IsZero() || c.mode != mode || age(c.written) > m.timing.renewAfter ignore := -1 if a, ok := m.adopted[recordID]; ok { ignore = a.slot @@ -350,7 +350,7 @@ func (m *Manager) claim(recordID uint64, key lockKey, mode LockMode) (written ti } if _, lower, by := m.rowConflicts(recordID, row, mode, ignore); lower { m.deferred.Add(1) - if !marked.IsZero() && age(marked) > renewAfter { + if !marked.IsZero() && age(marked) > m.timing.renewAfter { at, err := m.markWaiting(recordID, key) m.mu.Lock() c.marked = at @@ -446,7 +446,7 @@ func (m *Manager) releaseRecord(recordID uint64) error { return nil } if !m.unreleased[recordID] { - log.Printf("[WARN] lock: clearing this node's claim on record %d failed: %v; retrying every %v until it is cleared", recordID, err, releaseRetry) + log.Printf("[WARN] lock: clearing this node's claim on record %d failed: %v; retrying every %v until it is cleared", recordID, err, m.timing.releaseRetry) } m.unreleased[recordID] = true return err