diff --git a/internal/lock/fairness_test.go b/internal/lock/fairness_test.go index e0b7038c..df8c9263 100644 --- a/internal/lock/fairness_test.go +++ b/internal/lock/fairness_test.go @@ -80,11 +80,10 @@ func TestHolderYieldsOnlyToLiveWaiters(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - defer func(d time.Duration) { holdTurn = d }(holdTurn) - holdTurn = 50 * time.Millisecond dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() holder := newTestNode(dev, 0, 1<<20, 1) + holder.timing.holdTurn = 50 * time.Millisecond if tc.view { reg := newFakeRegistry() reg.set(5, tc.state) @@ -98,7 +97,7 @@ func TestHolderYieldsOnlyToLiveWaiters(t *testing.T) { stop := make(chan struct{}) var wg sync.WaitGroup wg.Go(func() { reclaimAfterExpiry(t, holder, LockTypeInode, 3, stop) }) - time.Sleep(4 * holdTurn) + time.Sleep(4 * holder.timing.holdTurn) close(stop) wg.Wait() diff --git a/internal/lock/holds.go b/internal/lock/holds.go index acf882fc..cae63238 100644 --- a/internal/lock/holds.go +++ b/internal/lock/holds.go @@ -9,15 +9,6 @@ import ( "time" ) -// longHold is how long a node holds a record before it warns that the hold may -// have leaked. Under an owner view a claim never goes stale, so a hold that is -// never released lasts until unmount. -var longHold = time.Minute - -// releaseRetry is how often the watcher clears again a slot whose release write -// failed. Other nodes wait for the record until it is cleared. -var releaseRetry = time.Second - // Start runs a watcher that logs a warning for each record this node has held // for more than longHold. Stop ends it. func (m *Manager) Start() { @@ -44,9 +35,9 @@ func (m *Manager) Stop() { func (m *Manager) watchHolds(stop <-chan struct{}, done chan<- struct{}) { defer close(done) - tick := time.NewTicker(longHold / 6) + tick := time.NewTicker(m.timing.longHold / 6) defer tick.Stop() - retry := time.NewTicker(releaseRetry) + retry := time.NewTicker(m.timing.releaseRetry) defer retry.Stop() for { select { @@ -66,7 +57,7 @@ func (m *Manager) warnLongHolds() { m.mu.Lock() var warnings []string for recordID, rec := range m.records { - if rec.keys == 0 || rec.warnedLong || age(rec.since) <= longHold { + if rec.keys == 0 || rec.warnedLong || age(rec.since) <= m.timing.longHold { continue } rec.warnedLong = true diff --git a/internal/lock/holds_test.go b/internal/lock/holds_test.go index 0f5aa7b9..7e0c217c 100644 --- a/internal/lock/holds_test.go +++ b/internal/lock/holds_test.go @@ -11,19 +11,17 @@ import ( ) func TestLongHoldIsLoggedOnce(t *testing.T) { - old := longHold - t.Cleanup(func() { longHold = old }) - longHold = 120 * time.Millisecond dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() holder := newTestNode(dev, 0, 1<<20, 1) + holder.timing.longHold = 120 * time.Millisecond var logs bytes.Buffer log.SetOutput(&logs) t.Cleanup(func() { log.SetOutput(os.Stderr) }) holder.Start() requireAcquired(t, holder, LockTypeInode, 2, LockModeExclusive) - time.Sleep(5 * longHold) + time.Sleep(5 * holder.timing.longHold) holder.Stop() log.SetOutput(os.Stderr) diff --git a/internal/lock/lock.go b/internal/lock/lock.go index f0d158de..c896bbbb 100644 --- a/internal/lock/lock.go +++ b/internal/lock/lock.go @@ -170,6 +170,7 @@ type Manager struct { onAcquire OnAcquireFunc waiting atomic.Int64 + timing timings slotWrites, rowReads, conflicts, waits, deferred atomic.Uint64 // claimTimes is a histogram of the time from a claim's slot write to the end of its row read. @@ -250,22 +251,39 @@ func covers(held, want LockMode) bool { return held == LockModeExclusive || held == want } -// renewAfter is the age of a slot after which this node writes it again before -// relying on it, so other nodes never see a slot in use as stale. -var renewAfter = StaleThreshold / 3 - -// acquireTimeout is how long Acquire waits at most; tests shorten it. -var acquireTimeout = DefaultTimeout - -// tornExpiry is how long a slot must keep failing its checksum before other -// nodes ignore it. A live node rewrites its slot within renewAfter. -var tornExpiry = StaleThreshold - -var ( - cacheGrace = CacheGrace - holdTurn = HoldTurn - yieldGap = YieldGap -) +// timings are a manager's timing settings. Tests shorten them on the manager +// they test, before using it, so no test changes what another test's timers read. +type timings struct { + cacheGrace, holdTurn, yieldGap time.Duration + // renewAfter is the age of a slot after which this node writes it again + // before relying on it, so other nodes never see a slot in use as stale. + renewAfter time.Duration + // acquireTimeout is how long Acquire waits at most. + acquireTimeout time.Duration + // tornExpiry is how long a slot must keep failing its checksum before other + // nodes ignore it. A live node rewrites its slot within renewAfter. + tornExpiry time.Duration + // longHold is how long a node holds a record before it warns that the hold + // may have leaked. Under an owner view a claim never goes stale, so a hold + // that is never released lasts until unmount. + longHold time.Duration + // releaseRetry is how often the watcher clears again a slot whose release + // write failed. Other nodes wait for the record until it is cleared. + releaseRetry time.Duration +} + +func defaultTimings() timings { + return timings{ + cacheGrace: CacheGrace, + holdTurn: HoldTurn, + yieldGap: YieldGap, + renewAfter: StaleThreshold / 3, + acquireTimeout: DefaultTimeout, + tornExpiry: StaleThreshold, + longHold: time.Minute, + releaseRetry: time.Second, + } +} // releaseParallelism is how many kept records ReleaseCached frees at once. const releaseParallelism = 8 @@ -301,6 +319,7 @@ func NewManager(dev *storage.Device, offset, size uint64, nodeUUID [16]byte, nod adopted: make(map[uint64]adoptedSlot), unreleased: make(map[uint64]bool), tornSince: make(map[slotRef]tornSighting), + timing: defaultTimings(), } } @@ -533,7 +552,7 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin return m.giveUp(ErrLockTimeout, lockType, resourceID, mode, startTime, fmt.Sprintf( "the journal replay of node %x (slot %d), which holds record %d, failed or was abandoned", by.occupant[:8], by.slot, recordID)) } - if by.failing() && time.Since(lastFailingLog) >= acquireTimeout { + if by.failing() && time.Since(lastFailingLog) >= m.timing.acquireTimeout { lastFailingLog = time.Now() log.Printf("[WARN] lock: type=%d res=%d mode=%d has waited %v for node %x (slot %d), which holds it and is failing or being recovered; waiting until its journal replay ends", lockType, resourceID, mode, time.Since(startTime).Round(time.Millisecond), by.occupant[:8], by.slot) @@ -542,7 +561,7 @@ func (m *Manager) Acquire(ctx context.Context, lockType LockType, resourceID uin // Check context cancellation, but allow at least 3 retries. A wait gives // up after acquireTimeout of waiting for live holders, or for failing ones // while this goroutine holds other locks; a replay's wait never does. - if retryCount >= 3 && (ctx.Err() != nil || !replay && liveWait > acquireTimeout) { + if retryCount >= 3 && (ctx.Err() != nil || !replay && liveWait > m.timing.acquireTimeout) { if failingWhileNested { return m.giveUp(ErrReplayBlocked, lockType, resourceID, mode, startTime, fmt.Sprintf( "this goroutine holds other locks, and record %d is held by node %x (slot %d), which is failing or being recovered and whose replay may need one of them", @@ -581,7 +600,7 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi defer gate.Unlock() m.mu.Lock() - if rec, ok := m.records[recordID]; ok && rec.keys == 0 && (!covers(rec.mode, mode) || age(rec.renewed) > renewAfter) { + if rec, ok := m.records[recordID]; ok && rec.keys == 0 && (!covers(rec.mode, mode) || age(rec.renewed) > m.timing.renewAfter) { // A cached hold too weak for this request, or old enough that another node // may soon ignore it as stale, is claimed afresh over this node's slot. delete(m.records, recordID) @@ -596,7 +615,7 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi return false, false, blocker{}, nil } if m.turnOverLocked(recordID, rec) && !m.goroutineHoldsRecordLocked(recordID, gid) && - (waited < holdTurn || !m.goroutineHoldsAnyLocked(gid)) { + (waited < m.timing.holdTurn || !m.goroutineHoldsAnyLocked(gid)) { if rec.keys > 0 { m.mu.Unlock() return false, false, blocker{}, nil @@ -605,7 +624,7 @@ func (m *Manager) attemptAcquire(key lockKey, recordID uint64, mode LockMode, gi m.mu.Unlock() return false, false, blocker{}, m.releaseRecord(recordID) } - renew := age(rec.renewed) > renewAfter + renew := age(rec.renewed) > m.timing.renewAfter m.mu.Unlock() if renew { if err := m.renewRecord(recordID, key, rec); err != nil { @@ -665,7 +684,7 @@ func (m *Manager) giveUp(err error, lockType LockType, resourceID uint64, mode L // saw another node waiting. func (m *Manager) turnOverLocked(recordID uint64, rec *heldRecord) bool { seen, waited := m.contested[recordID] - return age(rec.since) > holdTurn || waited && age(seen) > holdTurn + return age(rec.since) > m.timing.holdTurn || waited && age(seen) > m.timing.holdTurn } // endTurnLocked drops this node's hold on recordID, which the caller releases @@ -674,7 +693,7 @@ func (m *Manager) turnOverLocked(recordID uint64, rec *heldRecord) bool { func (m *Manager) endTurnLocked(recordID uint64) { delete(m.records, recordID) delete(m.contested, recordID) - m.yieldUntil[recordID] = time.Now().Add(yieldGap) + m.yieldUntil[recordID] = time.Now().Add(m.timing.yieldGap) } func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) bool { @@ -683,7 +702,7 @@ func (m *Manager) reuseHeldKeyLocked(key lockKey, mode LockMode, gid uint64) boo return false } rec := m.records[held.RecordID] - if !covers(rec.mode, mode) || age(rec.renewed) > renewAfter { + if !covers(rec.mode, mode) || age(rec.renewed) > m.timing.renewAfter { return false } if m.turnOverLocked(held.RecordID, rec) && !m.goroutineHoldsRecordLocked(held.RecordID, gid) { @@ -795,7 +814,7 @@ func (m *Manager) dropRecordKey(recordID uint64) error { gen := rec.cacheGen m.mu.Unlock() - time.AfterFunc(cacheGrace, func() { m.expireCached(recordID, gen) }) + time.AfterFunc(m.timing.cacheGrace, func() { m.expireCached(recordID, gen) }) return nil } diff --git a/internal/lock/owner_test.go b/internal/lock/owner_test.go index 5d3e57c5..9c1702c3 100644 --- a/internal/lock/owner_test.go +++ b/internal/lock/owner_test.go @@ -43,11 +43,10 @@ func (f *fakeRegistry) view(slot int, fresh bool) ([16]byte, OwnerState) { return [16]byte{}, OwnerReleased } -func shortenAcquireTimeout(t *testing.T, d time.Duration) { - t.Helper() - old := acquireTimeout - t.Cleanup(func() { acquireTimeout = old }) - acquireTimeout = d +// shortenAcquireTimeout sets d as the acquire timeout of ms and returns it. +func shortenAcquireTimeout(d time.Duration, ms ...*Manager) time.Duration { + setTimings(func(t *timings) { t.acquireTimeout = d }, ms...) + return d } // acquireInBackground starts an Acquire with no deadline and returns its result. @@ -96,29 +95,29 @@ func TestFailingHoldersOldClaimBlocksUntilRegistrySlotEmpties(t *testing.T) { } func TestWaitForFailingHolderOutlastsTimeout(t *testing.T) { - shortenAcquireTimeout(t, 300*time.Millisecond) dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() holder := newTestNode(dev, 0, 1<<20, 1) waiter := newTestNode(dev, 0, 1<<20, 2) + timeout := shortenAcquireTimeout(300*time.Millisecond, holder, waiter) reg := newFakeRegistry() reg.set(1, OwnerFailing) waiter.SetOwnerView(reg.view) requireAcquired(t, holder, LockTypeInode, 2, LockModeExclusive) result := acquireInBackground(waiter, LockTypeInode, 2) - requireWaiting(t, result, 3*acquireTimeout) + requireWaiting(t, result, 3*timeout) reg.set(1, OwnerReleased) requireResult(t, result, nil) } func TestWaitForLiveHolderStillTimesOut(t *testing.T) { - shortenAcquireTimeout(t, 300*time.Millisecond) dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() holder := newTestNode(dev, 0, 1<<20, 1) waiter := newTestNode(dev, 0, 1<<20, 2) + shortenAcquireTimeout(300*time.Millisecond, holder, waiter) reg := newFakeRegistry() reg.set(1, OwnerAlive) waiter.SetOwnerView(reg.view) @@ -179,12 +178,12 @@ func TestTornSlotFreesWhenItsRegistrySlotEmpties(t *testing.T) { // while it replays that journal at mount. Peers wait for the replay, however // long, though the node itself is alive. func TestPeersWaitOutLiveNodesReplayOfItsPredecessor(t *testing.T) { - shortenAcquireTimeout(t, 300*time.Millisecond) dev, cleanup := createTestDevice(t, reclaimRegion) defer cleanup() predecessor := NewManager(dev, 0, reclaimRegion, [16]byte{1}, 1) successor := NewManager(dev, 0, reclaimRegion, [16]byte{5}, 1) waiter := NewManager(dev, 0, reclaimRegion, [16]byte{2}, 2) + timeout := shortenAcquireTimeout(300*time.Millisecond, predecessor, successor, waiter) reg := newFakeRegistry() reg.set(1, OwnerAlive) waiter.SetOwnerView(reg.view) @@ -195,7 +194,7 @@ func TestPeersWaitOutLiveNodesReplayOfItsPredecessor(t *testing.T) { t.Fatalf("AdoptOwner = %v, %v; want the predecessor's record", adopted, err) } result := acquireInBackground(waiter, LockTypeInode, 2) - requireWaiting(t, result, 3*acquireTimeout) + requireWaiting(t, result, 3*timeout) if err := successor.ReleaseAdopted(adopted); err != nil { t.Fatal(err) @@ -247,12 +246,12 @@ func TestReplaysSharingRecordTakeTurns(t *testing.T) { // gives up after the timeout and lets Y go, and the replay, which waits without // a limit, takes Y and can finish. func TestNestedWaitOnReplayingNodeYieldsToReplay(t *testing.T) { - shortenAcquireTimeout(t, 300*time.Millisecond) dev, cleanup := createTestDevice(t, reclaimRegion) defer cleanup() dead := NewManager(dev, 0, reclaimRegion, [16]byte{1}, 1) replayer := NewManager(dev, 0, reclaimRegion, [16]byte{3}, 3) peer := NewManager(dev, 0, reclaimRegion, [16]byte{4}, 4) + timeout := shortenAcquireTimeout(300*time.Millisecond, dead, replayer, peer) reg := newFakeRegistry() reg.set(1, OwnerFailing) reg.set(3, OwnerAlive) @@ -294,13 +293,13 @@ func TestNestedWaitOnReplayingNodeYieldsToReplay(t *testing.T) { go func() { replayResult <- replayer.Acquire(ForReplay(context.Background()), LockTypeInode, inodeY, LockModeExclusive) }() - time.Sleep(acquireTimeout / 2) + time.Sleep(timeout / 2) close(askX) requireResult(t, peerResult, ErrLockTimeout) requireResult(t, replayResult, nil) - if waited := time.Since(replayStart); waited <= acquireTimeout { - t.Errorf("replay waited %v, want longer than the %v timeout", waited, acquireTimeout) + if waited := time.Since(replayStart); waited <= timeout { + t.Errorf("replay waited %v, want longer than the %v timeout", waited, timeout) } log.SetOutput(os.Stderr) if want := fmt.Sprintf("held by node %x (slot 1)", [8]byte{1}); !strings.Contains(logs.String(), want) { @@ -345,11 +344,11 @@ func TestReplayDoesNotPassOverLiveNodesReplayClaims(t *testing.T) { // A caller that can release its locks and retry gets ErrReplayBlocked at once, // instead of holding them for the timeout while a replay may need one. func TestRetryableNestedWaitOnFailingHolderReturnsAtOnce(t *testing.T) { - shortenAcquireTimeout(t, 5*time.Second) dev, cleanup := createTestDevice(t, reclaimRegion) defer cleanup() dead := NewManager(dev, 0, reclaimRegion, [16]byte{1}, 1) peer := NewManager(dev, 0, reclaimRegion, [16]byte{4}, 4) + shortenAcquireTimeout(5*time.Second, dead, peer) reg := newFakeRegistry() reg.set(1, OwnerFailing) reg.set(4, OwnerAlive) diff --git a/internal/lock/release_retry_test.go b/internal/lock/release_retry_test.go index a1e97e43..85940ada 100644 --- a/internal/lock/release_retry_test.go +++ b/internal/lock/release_retry_test.go @@ -31,7 +31,7 @@ func TestFailedReleaseWriteIsRetried(t *testing.T) { failNext.Store(true) release(t, a, LockTypeInode, 3) - if err := acquireWithin(b, 5*releaseRetry, LockTypeInode, 3, LockModeExclusive); err != nil { + if err := acquireWithin(b, 5*a.timing.releaseRetry, LockTypeInode, 3, LockModeExclusive); err != nil { t.Fatalf("the other node is still kept out after one failed release write: %v", err) } release(t, b, LockTypeInode, 3) diff --git a/internal/lock/safety_test.go b/internal/lock/safety_test.go index 87d9af84..a241d84b 100644 --- a/internal/lock/safety_test.go +++ b/internal/lock/safety_test.go @@ -23,6 +23,13 @@ func newTestNode(dev *storage.Device, offset, size uint64, n int) *Manager { return NewManager(dev, offset, size, uuid, n) } +// setTimings changes the timings of ms. Call it before ms are used. +func setTimings(set func(*timings), ms ...*Manager) { + for _, m := range ms { + set(&m.timing) + } +} + func acquireWithin(m *Manager, d time.Duration, lt LockType, id uint64, mode LockMode) error { ctx, cancel := context.WithTimeout(context.Background(), d) defer cancel() @@ -142,8 +149,7 @@ func TestConcurrentAcquireOnOneNodeKeepsOtherNodesOut(t *testing.T) { a := newTestNode(dev, 0, 1<<20, 1) b := newTestNode(dev, 0, 1<<20, 2) // The workers wait for each other while holding, so all must join the hold, however late the scheduler starts them. - defer func(d time.Duration) { holdTurn = d }(holdTurn) - holdTurn = time.Minute + setTimings(func(t *timings) { t.holdTurn = time.Minute }, a, b) const workers = 4 for iter := 0; iter < 30; iter++ { @@ -274,8 +280,7 @@ func TestJoiningKeyRenewsBusyRecord(t *testing.T) { defer cleanup() a := newTestNode(dev, 0, RowSize, 1) b := newTestNode(dev, 0, RowSize, 2) - defer func(d time.Duration) { renewAfter = d }(renewAfter) - renewAfter = 0 + setTimings(func(t *timings) { t.renewAfter = 0 }, a, b) requireAcquired(t, a, LockTypeInode, 1, LockModeExclusive) ageSlot(t, a, LockTypeInode, 1, pastStale) @@ -314,7 +319,7 @@ func TestPausedHolderReleaseKeepsReclaimedLock(t *testing.T) { requireAcquired(t, b, LockTypeInode, 3, LockModeExclusive) release(t, a, LockTypeInode, 3) - time.Sleep(cacheGrace + 50*time.Millisecond) + time.Sleep(a.timing.cacheGrace + 50*time.Millisecond) requireBlocked(t, c, LockTypeInode, 3, LockModeExclusive) } @@ -327,8 +332,6 @@ func TestLockRegionStaysBoundedAcrossCycles(t *testing.T) { ) dev, cleanup := createTestDevice(t, guard+regionSize+guard) defer cleanup() - defer func(g time.Duration) { cacheGrace = g }(cacheGrace) - cacheGrace = time.Millisecond sentinel := storage.NewAlignedBuffer(guard) for i := range sentinel.Bytes() { @@ -344,6 +347,7 @@ func TestLockRegionStaysBoundedAcrossCycles(t *testing.T) { newTestNode(dev, guard, regionSize, 1), newTestNode(dev, guard, regionSize, 2), } + setTimings(func(t *timings) { t.cacheGrace = time.Millisecond }, nodes...) var wg sync.WaitGroup errs := make(chan error, len(nodes)) for n, m := range nodes { @@ -396,8 +400,7 @@ func TestReleasedHoldIsCachedForGrace(t *testing.T) { defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) b := newTestNode(dev, 0, 1<<20, 2) - defer func(g time.Duration) { cacheGrace = g }(cacheGrace) - cacheGrace = 200 * time.Millisecond + setTimings(func(t *timings) { t.cacheGrace = 200 * time.Millisecond }, a, b) requireAcquired(t, a, LockTypeInode, 6, LockModeExclusive) if err := a.Release(LockTypeInode, 6); err != nil { @@ -407,8 +410,8 @@ func TestReleasedHoldIsCachedForGrace(t *testing.T) { start := time.Now() requireAcquired(t, b, LockTypeInode, 6, LockModeExclusive) - if waited := time.Since(start); waited > cacheGrace+4*RetryMaxDelay { - t.Fatalf("other node waited %v for a cached hold, want at most about the %v grace", waited, cacheGrace) + if waited := time.Since(start); waited > a.timing.cacheGrace+4*RetryMaxDelay { + t.Fatalf("other node waited %v for a cached hold, want at most about the %v grace", waited, a.timing.cacheGrace) } } @@ -424,8 +427,8 @@ func TestIdleCachedHoldFreesBeforeTurnEnds(t *testing.T) { } start := time.Now() requireAcquired(t, b, LockTypeInode, 7, LockModeExclusive) - if waited := time.Since(start); waited >= holdTurn { - t.Fatalf("other node waited %v for an idle hold, want less than the %v turn a busy holder gets", waited, holdTurn) + if waited := time.Since(start); waited >= a.timing.holdTurn { + t.Fatalf("other node waited %v for an idle hold, want less than the %v turn a busy holder gets", waited, a.timing.holdTurn) } } @@ -434,9 +437,7 @@ func TestPausedNodeDropsCachedHoldTakenOver(t *testing.T) { defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) b := newTestNode(dev, 0, 1<<20, 2) - defer func(d time.Duration) { renewAfter = d }(renewAfter) - defer func(g time.Duration) { cacheGrace = g }(cacheGrace) - cacheGrace = time.Minute + setTimings(func(t *timings) { t.cacheGrace = time.Minute }, a, b) requireAcquired(t, a, LockTypeInode, 8, LockModeExclusive) if err := a.Release(LockTypeInode, 8); err != nil { @@ -445,7 +446,7 @@ func TestPausedNodeDropsCachedHoldTakenOver(t *testing.T) { // A pauses past StaleThreshold with the write hold cached, and B takes the record over. ageSlot(t, a, LockTypeInode, 8, pastStale) requireAcquired(t, b, LockTypeInode, 8, LockModeExclusive) - renewAfter = 0 + a.timing.renewAfter = 0 requireBlocked(t, a, LockTypeInode, 8, LockModeExclusive) } @@ -455,8 +456,7 @@ func TestReleaseCachedFreesHoldsAtOnce(t *testing.T) { defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) b := newTestNode(dev, 0, 1<<20, 2) - defer func(g time.Duration) { cacheGrace = g }(cacheGrace) - cacheGrace = time.Minute + setTimings(func(t *timings) { t.cacheGrace = time.Minute }, a, b) requireAcquired(t, a, LockTypeInode, 9, LockModeExclusive) if err := a.Release(LockTypeInode, 9); err != nil { @@ -575,8 +575,7 @@ func TestReleaseCachedWithSlowWritesRunsInParallel(t *testing.T) { dev, cleanup := createTestDevice(t, region) defer cleanup() a := newTestNode(dev, 0, region, 1) - defer func(g time.Duration) { cacheGrace = g }(cacheGrace) - cacheGrace = time.Minute + a.timing.cacheGrace = time.Minute records := map[uint64]bool{} for id := uint64(100); len(records) < 16; id++ { @@ -622,8 +621,7 @@ func TestAcquireWithoutDeadlineGivesUp(t *testing.T) { defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) b := newTestNode(dev, 0, 1<<20, 2) - defer func(d time.Duration) { acquireTimeout = d }(acquireTimeout) - acquireTimeout = 200 * time.Millisecond + b.timing.acquireTimeout = 200 * time.Millisecond requireAcquired(t, a, LockTypeInode, 40, LockModeExclusive) errc := make(chan error, 1) @@ -768,13 +766,12 @@ func TestTornSlotCountsAsClaim(t *testing.T) { func TestTornSlotExpires(t *testing.T) { dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() - defer func(d time.Duration) { tornExpiry = d }(tornExpiry) - tornExpiry = 200 * time.Millisecond a := newTestNode(dev, 0, 1<<20, 1) + a.timing.tornExpiry = 200 * time.Millisecond tearSlot(t, a, LockTypeInode, 51, 2) requireBlocked(t, a, LockTypeInode, 51, LockModeExclusive) - time.Sleep(tornExpiry) + time.Sleep(a.timing.tornExpiry) requireAcquired(t, a, LockTypeInode, 51, LockModeExclusive) } @@ -784,13 +781,12 @@ func TestTornSlotExpires(t *testing.T) { func TestTornSlotWaitRestartsWhenBytesChange(t *testing.T) { dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() - defer func(d time.Duration) { tornExpiry = d }(tornExpiry) - tornExpiry = 200 * time.Millisecond a := newTestNode(dev, 0, 1<<20, 1) + a.timing.tornExpiry = 200 * time.Millisecond tearSlot(t, a, LockTypeInode, 51, 2) requireBlocked(t, a, LockTypeInode, 51, LockModeExclusive) - time.Sleep(tornExpiry) + time.Sleep(a.timing.tornExpiry) tearSlot(t, a, LockTypeInode, 51, 2) requireBlocked(t, a, LockTypeInode, 51, LockModeExclusive) } @@ -968,8 +964,7 @@ func TestCachedHoldRejoinNeedsNoDiskIO(t *testing.T) { dev, cleanup := createTestDevice(t, 1<<20) defer cleanup() a := newTestNode(dev, 0, 1<<20, 1) - defer func(g time.Duration) { cacheGrace = g }(cacheGrace) - cacheGrace = time.Minute + a.timing.cacheGrace = time.Minute requireAcquired(t, a, LockTypeInode, 7, LockModeExclusive) if err := a.Release(LockTypeInode, 7); err != nil { t.Fatalf("Release: %v", err) diff --git a/internal/lock/slots.go b/internal/lock/slots.go index ce3a3b6e..14db8520 100644 --- a/internal/lock/slots.go +++ b/internal/lock/slots.go @@ -265,7 +265,7 @@ func (m *Manager) tornTooLong(ref slotRef, torn []byte) bool { m.tornSince[ref] = tornSighting{since: time.Now(), sum: sum} return false } - return age(seen.since) > tornExpiry + return age(seen.since) > m.timing.tornExpiry } // ownAdoptedLocked reports whether AdoptOwner kept recordID in this node's own @@ -335,7 +335,7 @@ func (m *Manager) dropClaimWaiter(recordID uint64) { func (m *Manager) claim(recordID uint64, key lockKey, mode LockMode) (written time.Time, won bool, by blocker, err error) { m.mu.Lock() c := m.claims[recordID] - rewrite := c.written.IsZero() || c.mode != mode || age(c.written) > renewAfter + rewrite := c.written.IsZero() || c.mode != mode || age(c.written) > m.timing.renewAfter ignore := -1 if a, ok := m.adopted[recordID]; ok { ignore = a.slot @@ -350,7 +350,7 @@ func (m *Manager) claim(recordID uint64, key lockKey, mode LockMode) (written ti } if _, lower, by := m.rowConflicts(recordID, row, mode, ignore); lower { m.deferred.Add(1) - if !marked.IsZero() && age(marked) > renewAfter { + if !marked.IsZero() && age(marked) > m.timing.renewAfter { at, err := m.markWaiting(recordID, key) m.mu.Lock() c.marked = at @@ -446,7 +446,7 @@ func (m *Manager) releaseRecord(recordID uint64) error { return nil } if !m.unreleased[recordID] { - log.Printf("[WARN] lock: clearing this node's claim on record %d failed: %v; retrying every %v until it is cleared", recordID, err, releaseRetry) + log.Printf("[WARN] lock: clearing this node's claim on record %d failed: %v; retrying every %v until it is cleared", recordID, err, m.timing.releaseRetry) } m.unreleased[recordID] = true return err