diff --git a/services/_common/faults.py b/services/_common/faults.py index 40d477f..9874077 100644 --- a/services/_common/faults.py +++ b/services/_common/faults.py @@ -36,7 +36,17 @@ class FaultRequest(BaseModel): """Specification for an injected fault.""" kind: FaultKind - magnitude: float = Field(ge=0, description="Magnitude of fault (ms, percentage, MB, conns)") + magnitude: float = Field( + ge=0, + description=( + "Magnitude of fault, units depend on kind: LATENCY is milliseconds, " + "MEMORY_LEAK is megabytes, POOL_EXHAUSTION is a connection count, CPU_SPIN " + "is milliseconds of burn time. ERROR_RATE is the odd one out: a value " + "<= 1.0 is read as a fraction (0.5 = 50%), a value > 1.0 as a percentage " + "(50.0 = 50%) -- so a caller who means '1%' and passes 1.0 gets 100% " + "instead, since 1.0 lands on the fraction side of that boundary." + ), + ) ttl_seconds: float = Field(gt=0, description="Time-to-live before recovery in seconds") @@ -60,6 +70,14 @@ def __init__(self, pool: AsyncConnectionPool | None = None, seed: int | None = N self._held_connections: list[Any] = [] self._stall_event = asyncio.Event() self._stall_event.set() + # Guards the active-fault-transition critical section (active_fault, expires_at, + # _timer_task, _leaked_buffers, _held_connections) against two sources of + # concurrency: overlapping admin apply_fault/clear_fault calls, and the + # independent _ttl_watcher background task clearing a fault out from under a + # fresh apply_fault. Without it, e.g. a TTL expiry racing a new apply_fault could + # cancel the new fault's timer instead of the old one, or drop a connection + # POOL_EXHAUSTION just appended while clear_fault is mid-iteration over the list. + self._state_lock = asyncio.Lock() if seed is None: seed = int(os.getenv(FAULT_INJECTION_SEED_ENV, str(DEFAULT_FAULT_INJECTION_SEED))) self._rng = random.Random(seed) @@ -79,32 +97,40 @@ async def wait_if_stalled(self) -> None: async def apply_fault(self, fault: FaultRequest) -> None: """Apply a new fault and schedule TTL expiration.""" - await self.clear_fault() - - self.active_fault = fault - self.expires_at = time.time() + fault.ttl_seconds - - # Kind-specific initialization - if fault.kind == FaultKind.MEMORY_LEAK: - # Allocate magnitude in MB - self._leaked_buffers.append(bytearray(int(fault.magnitude * 1024 * 1024))) - elif fault.kind == FaultKind.POOL_EXHAUSTION and self.pool is not None: - # Acquire connections to exhaust pool - count = max(1, int(fault.magnitude)) - for _ in range(count): - try: - conn = await self.pool.getconn(timeout=1.0) - self._held_connections.append(conn) - except PoolTimeout: - break - elif fault.kind == FaultKind.STALL: - self._stall_event.clear() - - # Schedule automatic TTL clearance - self._timer_task = asyncio.create_task(self._ttl_watcher(fault.ttl_seconds)) + async with self._state_lock: + await self._clear_fault_locked() + + self.active_fault = fault + self.expires_at = time.time() + fault.ttl_seconds + + # Kind-specific initialization + if fault.kind == FaultKind.MEMORY_LEAK: + # Allocate magnitude in MB + self._leaked_buffers.append(bytearray(int(fault.magnitude * 1024 * 1024))) + elif fault.kind == FaultKind.POOL_EXHAUSTION and self.pool is not None: + # Acquire connections to exhaust pool + count = max(1, int(fault.magnitude)) + for _ in range(count): + try: + conn = await self.pool.getconn(timeout=1.0) + self._held_connections.append(conn) + except PoolTimeout: + break + elif fault.kind == FaultKind.STALL: + self._stall_event.clear() + + # Schedule automatic TTL clearance + self._timer_task = asyncio.create_task(self._ttl_watcher(fault.ttl_seconds)) async def clear_fault(self) -> None: """Revert active faults and restore baseline state.""" + async with self._state_lock: + await self._clear_fault_locked() + + async def _clear_fault_locked(self) -> None: + """Revert active faults and restore baseline state. Caller must hold + _state_lock; apply_fault calls this directly (already holding the lock) instead + of the public clear_fault to avoid a self-deadlock on the non-reentrant Lock.""" if self._timer_task and not self._timer_task.done(): self._timer_task.cancel() with contextlib.suppress(asyncio.CancelledError): @@ -147,6 +173,10 @@ async def pre_request_hook(self) -> None: await asyncio.sleep(fault.magnitude / 1000.0) elif fault.kind == FaultKind.ERROR_RATE: + # magnitude is a fraction at <= 1.0, a percentage above it (see FaultRequest + # .magnitude docstring) -- this boundary is intentional and load-bearing for + # test_faults.py::test_fault_error_rate (1.0 and 100.0 both mean 100%), not a + # bug to "fix" by changing the comparison. rate = fault.magnitude if fault.magnitude <= 1.0 else (fault.magnitude / 100.0) if self._rng.random() < rate: raise HTTPException( diff --git a/services/worker/app.py b/services/worker/app.py index 633e783..700eab9 100644 --- a/services/worker/app.py +++ b/services/worker/app.py @@ -10,7 +10,7 @@ from contextlib import asynccontextmanager from typing import Any -from fastapi import FastAPI +from fastapi import FastAPI, HTTPException from psycopg import Error as PsycopgError from psycopg_pool import AsyncConnectionPool @@ -96,12 +96,31 @@ async def process_one_job(pool: AsyncConnectionPool | None) -> bool: return False +async def _worker_iteration() -> None: + """Run one poll-and-process cycle, honoring whatever fault is currently active. + + Factored out of worker_loop so it can be exercised directly under test without + driving a real polling loop. pre_request_hook's ERROR_RATE branch raises + HTTPException, which is meaningful in an HTTP handler but has no response to attach + to here; treat it the same way a real job-processing failure is treated (log and + skip this iteration's job) rather than letting it crash the polling task. + """ + await fault_manager.wait_if_stalled() + try: + await fault_manager.pre_request_hook() + except HTTPException as exc: + logger.warning("worker_fault_injected", detail=exc.detail) + return + await process_one_job(db_pool) + + async def worker_loop() -> None: - """Continuous polling loop that honors active stall faults.""" + """Continuous polling loop that honors active faults (stall, latency, error_rate, + cpu_spin) applied via /admin/fault. Previously only consulted wait_if_stalled, so + every other fault kind was a no-op against job processing.""" while True: try: - await fault_manager.wait_if_stalled() - await process_one_job(db_pool) + await _worker_iteration() except asyncio.CancelledError: break await asyncio.sleep(POLL_INTERVAL_SECONDS) diff --git a/tests/unit/services/test_faults.py b/tests/unit/services/test_faults.py index 624992a..fa07171 100644 --- a/tests/unit/services/test_faults.py +++ b/tests/unit/services/test_faults.py @@ -188,6 +188,45 @@ async def test_fault_pool_exhaustion_errors_handled() -> None: assert len(manager._held_connections) == 0 +@pytest.mark.asyncio +async def test_fault_concurrent_apply_and_clear_stays_consistent() -> None: + """Concurrent admin apply_fault/clear_fault calls (or one racing the independent + _ttl_watcher background task) must never leave active_fault/expires_at/_timer_task + in a torn state -- the transition is guarded by an asyncio.Lock precisely to + prevent a fault that is "half applied" (e.g. active_fault set but no timer + scheduled yet) from being observed by a concurrent caller.""" + manager = FaultManager() + + async def apply(ttl: float) -> None: + await manager.apply_fault( + FaultRequest(kind=FaultKind.LATENCY, magnitude=1, ttl_seconds=ttl) + ) + + await asyncio.gather( + apply(5.0), + apply(6.0), + manager.clear_fault(), + apply(7.0), + manager.clear_fault(), + apply(8.0), + ) + + # Whatever interleaving the event loop chose, the manager must be internally + # consistent afterward: either fully cleared, or fully applied -- never a mix. + if manager.active_fault is None: + assert manager.expires_at is None + assert manager._timer_task is None or manager._timer_task.done() + else: + assert manager.expires_at is not None + assert manager._timer_task is not None + assert not manager._timer_task.done() + + await manager.clear_fault() + assert manager.active_fault is None + assert manager.expires_at is None + assert manager._timer_task is None + + @pytest.mark.asyncio async def test_fault_stall() -> None: """STALL blocks pre_request_hook indefinitely (like the worker), ignoring magnitude, diff --git a/tests/unit/services/test_worker.py b/tests/unit/services/test_worker.py index 2fa95a7..e1fc323 100644 --- a/tests/unit/services/test_worker.py +++ b/tests/unit/services/test_worker.py @@ -9,9 +9,12 @@ import pytest import services.worker.app as worker_module +from services._common.faults import FaultKind, FaultRequest from services.worker.app import ( + _worker_iteration, app, check_db_readiness, + fault_manager, init_worker_db, process_one_job, worker_loop, @@ -110,6 +113,35 @@ async def test_worker_loop_propagates_unexpected_errors() -> None: await worker_loop() +@pytest.mark.asyncio +async def test_worker_iteration_applies_error_rate_fault() -> None: + """Previously worker_loop never called pre_request_hook, so ERROR_RATE (and + LATENCY/CPU_SPIN) faults were no-ops against job processing -- only STALL worked, via + the separate wait_if_stalled call. A magnitude of 1.0 always triggers (see + faults.py's fraction/percentage convention), so the job must be skipped, not + processed, on every iteration while the fault is active.""" + worker_module.IN_MEMORY_JOBS = [{"id": 1, "payload": "task", "status": "pending"}] + await fault_manager.apply_fault( + FaultRequest(kind=FaultKind.ERROR_RATE, magnitude=1.0, ttl_seconds=10) + ) + try: + with patch("services.worker.app.process_one_job") as mock_process: + await _worker_iteration() + mock_process.assert_not_called() + assert worker_module.IN_MEMORY_JOBS[0]["status"] == "pending" + finally: + await fault_manager.clear_fault() + + +@pytest.mark.asyncio +async def test_worker_iteration_no_fault_processes_job() -> None: + """With no active fault, pre_request_hook is a no-op and the job is processed exactly + as before this fault-injection gap was fixed.""" + worker_module.IN_MEMORY_JOBS = [{"id": 1, "payload": "task", "status": "pending"}] + await _worker_iteration() + assert worker_module.IN_MEMORY_JOBS[0]["status"] == "completed" + + @pytest.mark.asyncio async def test_check_db_readiness_no_pool() -> None: worker_module.db_pool = None