From 3533ea1d00c688d160783cb1ebfc97198707a034 Mon Sep 17 00:00:00 2001 From: Rahul Sanghvi Date: Fri, 11 Sep 2026 00:00:30 -0400 Subject: [PATCH 1/3] fix(worker): apply pre_request_hook to job processing worker_loop only called fault_manager.wait_if_stalled, so LATENCY, ERROR_RATE, and CPU_SPIN faults applied via POST /admin/fault were no-ops against job processing -- only STALL worked. Every other service's HTTP middleware calls pre_request_hook once per request; the worker has no HTTP request in its polling loop, so call it directly once per iteration instead. Factor the loop body into _worker_iteration so it can be exercised directly under test without driving a real polling loop, and so pre_request_hook's ERROR_RATE branch (which raises HTTPException -- meaningful in an HTTP handler, not here) can be handled the same way a real job-processing failure already is: logged and the job skipped for that iteration, rather than crashing the polling task. Implements build-plan step A1.2 (fix on an already-completed service). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01CzBeBUsZmE38K1kWeLMhNh --- services/worker/app.py | 27 +++++++++++++++++++++---- tests/unit/services/test_worker.py | 32 ++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/services/worker/app.py b/services/worker/app.py index 633e783..700eab9 100644 --- a/services/worker/app.py +++ b/services/worker/app.py @@ -10,7 +10,7 @@ from contextlib import asynccontextmanager from typing import Any -from fastapi import FastAPI +from fastapi import FastAPI, HTTPException from psycopg import Error as PsycopgError from psycopg_pool import AsyncConnectionPool @@ -96,12 +96,31 @@ async def process_one_job(pool: AsyncConnectionPool | None) -> bool: return False +async def _worker_iteration() -> None: + """Run one poll-and-process cycle, honoring whatever fault is currently active. + + Factored out of worker_loop so it can be exercised directly under test without + driving a real polling loop. pre_request_hook's ERROR_RATE branch raises + HTTPException, which is meaningful in an HTTP handler but has no response to attach + to here; treat it the same way a real job-processing failure is treated (log and + skip this iteration's job) rather than letting it crash the polling task. + """ + await fault_manager.wait_if_stalled() + try: + await fault_manager.pre_request_hook() + except HTTPException as exc: + logger.warning("worker_fault_injected", detail=exc.detail) + return + await process_one_job(db_pool) + + async def worker_loop() -> None: - """Continuous polling loop that honors active stall faults.""" + """Continuous polling loop that honors active faults (stall, latency, error_rate, + cpu_spin) applied via /admin/fault. Previously only consulted wait_if_stalled, so + every other fault kind was a no-op against job processing.""" while True: try: - await fault_manager.wait_if_stalled() - await process_one_job(db_pool) + await _worker_iteration() except asyncio.CancelledError: break await asyncio.sleep(POLL_INTERVAL_SECONDS) diff --git a/tests/unit/services/test_worker.py b/tests/unit/services/test_worker.py index 2fa95a7..e1fc323 100644 --- a/tests/unit/services/test_worker.py +++ b/tests/unit/services/test_worker.py @@ -9,9 +9,12 @@ import pytest import services.worker.app as worker_module +from services._common.faults import FaultKind, FaultRequest from services.worker.app import ( + _worker_iteration, app, check_db_readiness, + fault_manager, init_worker_db, process_one_job, worker_loop, @@ -110,6 +113,35 @@ async def test_worker_loop_propagates_unexpected_errors() -> None: await worker_loop() +@pytest.mark.asyncio +async def test_worker_iteration_applies_error_rate_fault() -> None: + """Previously worker_loop never called pre_request_hook, so ERROR_RATE (and + LATENCY/CPU_SPIN) faults were no-ops against job processing -- only STALL worked, via + the separate wait_if_stalled call. A magnitude of 1.0 always triggers (see + faults.py's fraction/percentage convention), so the job must be skipped, not + processed, on every iteration while the fault is active.""" + worker_module.IN_MEMORY_JOBS = [{"id": 1, "payload": "task", "status": "pending"}] + await fault_manager.apply_fault( + FaultRequest(kind=FaultKind.ERROR_RATE, magnitude=1.0, ttl_seconds=10) + ) + try: + with patch("services.worker.app.process_one_job") as mock_process: + await _worker_iteration() + mock_process.assert_not_called() + assert worker_module.IN_MEMORY_JOBS[0]["status"] == "pending" + finally: + await fault_manager.clear_fault() + + +@pytest.mark.asyncio +async def test_worker_iteration_no_fault_processes_job() -> None: + """With no active fault, pre_request_hook is a no-op and the job is processed exactly + as before this fault-injection gap was fixed.""" + worker_module.IN_MEMORY_JOBS = [{"id": 1, "payload": "task", "status": "pending"}] + await _worker_iteration() + assert worker_module.IN_MEMORY_JOBS[0]["status"] == "completed" + + @pytest.mark.asyncio async def test_check_db_readiness_no_pool() -> None: worker_module.db_pool = None From 3d18eac6cb8c087fb68bdfc97e93ce266168bf92 Mon Sep 17 00:00:00 2001 From: Rahul Sanghvi Date: Fri, 11 Sep 2026 00:01:41 -0400 Subject: [PATCH 2/3] fix(services): guard fault-manager state transitions with a lock FaultManager.apply_fault/clear_fault mutate shared state (active_fault, expires_at, _timer_task, _leaked_buffers, _held_connections) across multiple await points with no synchronization. Two admin apply_fault/clear_fault calls racing each other, or the independent _ttl_watcher background task clearing a fault out from under a fresh apply_fault, could leave the manager in a torn state -- e.g. a TTL expiry cancelling the wrong timer, or a connection appended to _held_connections during clear_fault's iteration over the same list. Add an asyncio.Lock guarding the whole state transition. apply_fault internally calls clear_fault's logic, so factor that logic into _clear_fault_locked (no lock acquisition) called by both the public clear_fault (which acquires the lock) and apply_fault (which acquires the lock once and calls the internal version), avoiding a self-deadlock on the non-reentrant lock. Route _ttl_watcher's clear through the same locked path so it can't race a fresh apply_fault. Implements build-plan step A1.3 (fix on an already-completed service). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01CzBeBUsZmE38K1kWeLMhNh --- services/_common/faults.py | 62 +++++++++++++++++++----------- tests/unit/services/test_faults.py | 39 +++++++++++++++++++ 2 files changed, 78 insertions(+), 23 deletions(-) diff --git a/services/_common/faults.py b/services/_common/faults.py index 40d477f..8c3ad55 100644 --- a/services/_common/faults.py +++ b/services/_common/faults.py @@ -60,6 +60,14 @@ def __init__(self, pool: AsyncConnectionPool | None = None, seed: int | None = N self._held_connections: list[Any] = [] self._stall_event = asyncio.Event() self._stall_event.set() + # Guards the active-fault-transition critical section (active_fault, expires_at, + # _timer_task, _leaked_buffers, _held_connections) against two sources of + # concurrency: overlapping admin apply_fault/clear_fault calls, and the + # independent _ttl_watcher background task clearing a fault out from under a + # fresh apply_fault. Without it, e.g. a TTL expiry racing a new apply_fault could + # cancel the new fault's timer instead of the old one, or drop a connection + # POOL_EXHAUSTION just appended while clear_fault is mid-iteration over the list. + self._state_lock = asyncio.Lock() if seed is None: seed = int(os.getenv(FAULT_INJECTION_SEED_ENV, str(DEFAULT_FAULT_INJECTION_SEED))) self._rng = random.Random(seed) @@ -79,32 +87,40 @@ async def wait_if_stalled(self) -> None: async def apply_fault(self, fault: FaultRequest) -> None: """Apply a new fault and schedule TTL expiration.""" - await self.clear_fault() - - self.active_fault = fault - self.expires_at = time.time() + fault.ttl_seconds - - # Kind-specific initialization - if fault.kind == FaultKind.MEMORY_LEAK: - # Allocate magnitude in MB - self._leaked_buffers.append(bytearray(int(fault.magnitude * 1024 * 1024))) - elif fault.kind == FaultKind.POOL_EXHAUSTION and self.pool is not None: - # Acquire connections to exhaust pool - count = max(1, int(fault.magnitude)) - for _ in range(count): - try: - conn = await self.pool.getconn(timeout=1.0) - self._held_connections.append(conn) - except PoolTimeout: - break - elif fault.kind == FaultKind.STALL: - self._stall_event.clear() - - # Schedule automatic TTL clearance - self._timer_task = asyncio.create_task(self._ttl_watcher(fault.ttl_seconds)) + async with self._state_lock: + await self._clear_fault_locked() + + self.active_fault = fault + self.expires_at = time.time() + fault.ttl_seconds + + # Kind-specific initialization + if fault.kind == FaultKind.MEMORY_LEAK: + # Allocate magnitude in MB + self._leaked_buffers.append(bytearray(int(fault.magnitude * 1024 * 1024))) + elif fault.kind == FaultKind.POOL_EXHAUSTION and self.pool is not None: + # Acquire connections to exhaust pool + count = max(1, int(fault.magnitude)) + for _ in range(count): + try: + conn = await self.pool.getconn(timeout=1.0) + self._held_connections.append(conn) + except PoolTimeout: + break + elif fault.kind == FaultKind.STALL: + self._stall_event.clear() + + # Schedule automatic TTL clearance + self._timer_task = asyncio.create_task(self._ttl_watcher(fault.ttl_seconds)) async def clear_fault(self) -> None: """Revert active faults and restore baseline state.""" + async with self._state_lock: + await self._clear_fault_locked() + + async def _clear_fault_locked(self) -> None: + """Revert active faults and restore baseline state. Caller must hold + _state_lock; apply_fault calls this directly (already holding the lock) instead + of the public clear_fault to avoid a self-deadlock on the non-reentrant Lock.""" if self._timer_task and not self._timer_task.done(): self._timer_task.cancel() with contextlib.suppress(asyncio.CancelledError): diff --git a/tests/unit/services/test_faults.py b/tests/unit/services/test_faults.py index 624992a..fa07171 100644 --- a/tests/unit/services/test_faults.py +++ b/tests/unit/services/test_faults.py @@ -188,6 +188,45 @@ async def test_fault_pool_exhaustion_errors_handled() -> None: assert len(manager._held_connections) == 0 +@pytest.mark.asyncio +async def test_fault_concurrent_apply_and_clear_stays_consistent() -> None: + """Concurrent admin apply_fault/clear_fault calls (or one racing the independent + _ttl_watcher background task) must never leave active_fault/expires_at/_timer_task + in a torn state -- the transition is guarded by an asyncio.Lock precisely to + prevent a fault that is "half applied" (e.g. active_fault set but no timer + scheduled yet) from being observed by a concurrent caller.""" + manager = FaultManager() + + async def apply(ttl: float) -> None: + await manager.apply_fault( + FaultRequest(kind=FaultKind.LATENCY, magnitude=1, ttl_seconds=ttl) + ) + + await asyncio.gather( + apply(5.0), + apply(6.0), + manager.clear_fault(), + apply(7.0), + manager.clear_fault(), + apply(8.0), + ) + + # Whatever interleaving the event loop chose, the manager must be internally + # consistent afterward: either fully cleared, or fully applied -- never a mix. + if manager.active_fault is None: + assert manager.expires_at is None + assert manager._timer_task is None or manager._timer_task.done() + else: + assert manager.expires_at is not None + assert manager._timer_task is not None + assert not manager._timer_task.done() + + await manager.clear_fault() + assert manager.active_fault is None + assert manager.expires_at is None + assert manager._timer_task is None + + @pytest.mark.asyncio async def test_fault_stall() -> None: """STALL blocks pre_request_hook indefinitely (like the worker), ignoring magnitude, From 86c4abe94619215ca188cdd39db1eca1444008b4 Mon Sep 17 00:00:00 2001 From: Rahul Sanghvi Date: Fri, 11 Sep 2026 00:02:56 -0400 Subject: [PATCH 3/3] docs(services): clarify ERROR_RATE magnitude fraction/percentage convention FaultRequest.magnitude means something different per FaultKind, and for ERROR_RATE specifically it silently overloads two conventions: <= 1.0 is a fraction, > 1.0 is a percentage. A caller who means "1%" and passes 1.0 gets 100% instead, since 1.0 lands on the fraction side of that boundary. This is intentional, existing behavior (test_faults.py::test_fault_error_rate already asserts 1.0 and 100.0 both mean 100%) -- not changed here, only documented: an inline comment at the branch, and an expanded field docstring spelling out the per-kind units and the footgun. Implements build-plan step A1.3 (documentation follow-up on an already-completed service). Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01CzBeBUsZmE38K1kWeLMhNh --- services/_common/faults.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/services/_common/faults.py b/services/_common/faults.py index 8c3ad55..9874077 100644 --- a/services/_common/faults.py +++ b/services/_common/faults.py @@ -36,7 +36,17 @@ class FaultRequest(BaseModel): """Specification for an injected fault.""" kind: FaultKind - magnitude: float = Field(ge=0, description="Magnitude of fault (ms, percentage, MB, conns)") + magnitude: float = Field( + ge=0, + description=( + "Magnitude of fault, units depend on kind: LATENCY is milliseconds, " + "MEMORY_LEAK is megabytes, POOL_EXHAUSTION is a connection count, CPU_SPIN " + "is milliseconds of burn time. ERROR_RATE is the odd one out: a value " + "<= 1.0 is read as a fraction (0.5 = 50%), a value > 1.0 as a percentage " + "(50.0 = 50%) -- so a caller who means '1%' and passes 1.0 gets 100% " + "instead, since 1.0 lands on the fraction side of that boundary." + ), + ) ttl_seconds: float = Field(gt=0, description="Time-to-live before recovery in seconds") @@ -163,6 +173,10 @@ async def pre_request_hook(self) -> None: await asyncio.sleep(fault.magnitude / 1000.0) elif fault.kind == FaultKind.ERROR_RATE: + # magnitude is a fraction at <= 1.0, a percentage above it (see FaultRequest + # .magnitude docstring) -- this boundary is intentional and load-bearing for + # test_faults.py::test_fault_error_rate (1.0 and 100.0 both mean 100%), not a + # bug to "fix" by changing the comparison. rate = fault.magnitude if fault.magnitude <= 1.0 else (fault.magnitude / 100.0) if self._rng.random() < rate: raise HTTPException(