Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 54 additions & 24 deletions services/_common/faults.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,17 @@ class FaultRequest(BaseModel):
"""Specification for an injected fault."""

kind: FaultKind
magnitude: float = Field(ge=0, description="Magnitude of fault (ms, percentage, MB, conns)")
magnitude: float = Field(
ge=0,
description=(
"Magnitude of fault, units depend on kind: LATENCY is milliseconds, "
"MEMORY_LEAK is megabytes, POOL_EXHAUSTION is a connection count, CPU_SPIN "
"is milliseconds of burn time. ERROR_RATE is the odd one out: a value "
"<= 1.0 is read as a fraction (0.5 = 50%), a value > 1.0 as a percentage "
"(50.0 = 50%) -- so a caller who means '1%' and passes 1.0 gets 100% "
"instead, since 1.0 lands on the fraction side of that boundary."
),
)
ttl_seconds: float = Field(gt=0, description="Time-to-live before recovery in seconds")


Expand All @@ -60,6 +70,14 @@ def __init__(self, pool: AsyncConnectionPool | None = None, seed: int | None = N
self._held_connections: list[Any] = []
self._stall_event = asyncio.Event()
self._stall_event.set()
# Guards the active-fault-transition critical section (active_fault, expires_at,
# _timer_task, _leaked_buffers, _held_connections) against two sources of
# concurrency: overlapping admin apply_fault/clear_fault calls, and the
# independent _ttl_watcher background task clearing a fault out from under a
# fresh apply_fault. Without it, e.g. a TTL expiry racing a new apply_fault could
# cancel the new fault's timer instead of the old one, or drop a connection
# POOL_EXHAUSTION just appended while clear_fault is mid-iteration over the list.
self._state_lock = asyncio.Lock()
if seed is None:
seed = int(os.getenv(FAULT_INJECTION_SEED_ENV, str(DEFAULT_FAULT_INJECTION_SEED)))
self._rng = random.Random(seed)
Expand All @@ -79,32 +97,40 @@ async def wait_if_stalled(self) -> None:

async def apply_fault(self, fault: FaultRequest) -> None:
"""Apply a new fault and schedule TTL expiration."""
await self.clear_fault()

self.active_fault = fault
self.expires_at = time.time() + fault.ttl_seconds

# Kind-specific initialization
if fault.kind == FaultKind.MEMORY_LEAK:
# Allocate magnitude in MB
self._leaked_buffers.append(bytearray(int(fault.magnitude * 1024 * 1024)))
elif fault.kind == FaultKind.POOL_EXHAUSTION and self.pool is not None:
# Acquire connections to exhaust pool
count = max(1, int(fault.magnitude))
for _ in range(count):
try:
conn = await self.pool.getconn(timeout=1.0)
self._held_connections.append(conn)
except PoolTimeout:
break
elif fault.kind == FaultKind.STALL:
self._stall_event.clear()

# Schedule automatic TTL clearance
self._timer_task = asyncio.create_task(self._ttl_watcher(fault.ttl_seconds))
async with self._state_lock:
await self._clear_fault_locked()

self.active_fault = fault
self.expires_at = time.time() + fault.ttl_seconds

# Kind-specific initialization
if fault.kind == FaultKind.MEMORY_LEAK:
# Allocate magnitude in MB
self._leaked_buffers.append(bytearray(int(fault.magnitude * 1024 * 1024)))
elif fault.kind == FaultKind.POOL_EXHAUSTION and self.pool is not None:
# Acquire connections to exhaust pool
count = max(1, int(fault.magnitude))
for _ in range(count):
try:
conn = await self.pool.getconn(timeout=1.0)
self._held_connections.append(conn)
except PoolTimeout:
break
elif fault.kind == FaultKind.STALL:
self._stall_event.clear()

# Schedule automatic TTL clearance
self._timer_task = asyncio.create_task(self._ttl_watcher(fault.ttl_seconds))

async def clear_fault(self) -> None:
"""Revert active faults and restore baseline state."""
async with self._state_lock:
await self._clear_fault_locked()

async def _clear_fault_locked(self) -> None:
"""Revert active faults and restore baseline state. Caller must hold
_state_lock; apply_fault calls this directly (already holding the lock) instead
of the public clear_fault to avoid a self-deadlock on the non-reentrant Lock."""
if self._timer_task and not self._timer_task.done():
self._timer_task.cancel()
with contextlib.suppress(asyncio.CancelledError):
Expand Down Expand Up @@ -147,6 +173,10 @@ async def pre_request_hook(self) -> None:
await asyncio.sleep(fault.magnitude / 1000.0)

elif fault.kind == FaultKind.ERROR_RATE:
# magnitude is a fraction at <= 1.0, a percentage above it (see FaultRequest
# .magnitude docstring) -- this boundary is intentional and load-bearing for
# test_faults.py::test_fault_error_rate (1.0 and 100.0 both mean 100%), not a
# bug to "fix" by changing the comparison.
rate = fault.magnitude if fault.magnitude <= 1.0 else (fault.magnitude / 100.0)
if self._rng.random() < rate:
raise HTTPException(
Expand Down
27 changes: 23 additions & 4 deletions services/worker/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
from contextlib import asynccontextmanager
from typing import Any

from fastapi import FastAPI
from fastapi import FastAPI, HTTPException
from psycopg import Error as PsycopgError
from psycopg_pool import AsyncConnectionPool

Expand Down Expand Up @@ -96,12 +96,31 @@ async def process_one_job(pool: AsyncConnectionPool | None) -> bool:
return False


async def _worker_iteration() -> None:
"""Run one poll-and-process cycle, honoring whatever fault is currently active.

Factored out of worker_loop so it can be exercised directly under test without
driving a real polling loop. pre_request_hook's ERROR_RATE branch raises
HTTPException, which is meaningful in an HTTP handler but has no response to attach
to here; treat it the same way a real job-processing failure is treated (log and
skip this iteration's job) rather than letting it crash the polling task.
"""
await fault_manager.wait_if_stalled()
try:
await fault_manager.pre_request_hook()
except HTTPException as exc:
logger.warning("worker_fault_injected", detail=exc.detail)
return
await process_one_job(db_pool)


async def worker_loop() -> None:
"""Continuous polling loop that honors active stall faults."""
"""Continuous polling loop that honors active faults (stall, latency, error_rate,
cpu_spin) applied via /admin/fault. Previously only consulted wait_if_stalled, so
every other fault kind was a no-op against job processing."""
while True:
try:
await fault_manager.wait_if_stalled()
await process_one_job(db_pool)
await _worker_iteration()
except asyncio.CancelledError:
break
await asyncio.sleep(POLL_INTERVAL_SECONDS)
Expand Down
39 changes: 39 additions & 0 deletions tests/unit/services/test_faults.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,45 @@ async def test_fault_pool_exhaustion_errors_handled() -> None:
assert len(manager._held_connections) == 0


@pytest.mark.asyncio
async def test_fault_concurrent_apply_and_clear_stays_consistent() -> None:
"""Concurrent admin apply_fault/clear_fault calls (or one racing the independent
_ttl_watcher background task) must never leave active_fault/expires_at/_timer_task
in a torn state -- the transition is guarded by an asyncio.Lock precisely to
prevent a fault that is "half applied" (e.g. active_fault set but no timer
scheduled yet) from being observed by a concurrent caller."""
manager = FaultManager()

async def apply(ttl: float) -> None:
await manager.apply_fault(
FaultRequest(kind=FaultKind.LATENCY, magnitude=1, ttl_seconds=ttl)
)

await asyncio.gather(
apply(5.0),
apply(6.0),
manager.clear_fault(),
apply(7.0),
manager.clear_fault(),
apply(8.0),
)

# Whatever interleaving the event loop chose, the manager must be internally
# consistent afterward: either fully cleared, or fully applied -- never a mix.
if manager.active_fault is None:
assert manager.expires_at is None
assert manager._timer_task is None or manager._timer_task.done()
else:
assert manager.expires_at is not None
assert manager._timer_task is not None
assert not manager._timer_task.done()

await manager.clear_fault()
assert manager.active_fault is None
assert manager.expires_at is None
assert manager._timer_task is None


@pytest.mark.asyncio
async def test_fault_stall() -> None:
"""STALL blocks pre_request_hook indefinitely (like the worker), ignoring magnitude,
Expand Down
32 changes: 32 additions & 0 deletions tests/unit/services/test_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@
import pytest

import services.worker.app as worker_module
from services._common.faults import FaultKind, FaultRequest
from services.worker.app import (
_worker_iteration,
app,
check_db_readiness,
fault_manager,
init_worker_db,
process_one_job,
worker_loop,
Expand Down Expand Up @@ -110,6 +113,35 @@ async def test_worker_loop_propagates_unexpected_errors() -> None:
await worker_loop()


@pytest.mark.asyncio
async def test_worker_iteration_applies_error_rate_fault() -> None:
"""Previously worker_loop never called pre_request_hook, so ERROR_RATE (and
LATENCY/CPU_SPIN) faults were no-ops against job processing -- only STALL worked, via
the separate wait_if_stalled call. A magnitude of 1.0 always triggers (see
faults.py's fraction/percentage convention), so the job must be skipped, not
processed, on every iteration while the fault is active."""
worker_module.IN_MEMORY_JOBS = [{"id": 1, "payload": "task", "status": "pending"}]
await fault_manager.apply_fault(
FaultRequest(kind=FaultKind.ERROR_RATE, magnitude=1.0, ttl_seconds=10)
)
try:
with patch("services.worker.app.process_one_job") as mock_process:
await _worker_iteration()
mock_process.assert_not_called()
assert worker_module.IN_MEMORY_JOBS[0]["status"] == "pending"
finally:
await fault_manager.clear_fault()


@pytest.mark.asyncio
async def test_worker_iteration_no_fault_processes_job() -> None:
"""With no active fault, pre_request_hook is a no-op and the job is processed exactly
as before this fault-injection gap was fixed."""
worker_module.IN_MEMORY_JOBS = [{"id": 1, "payload": "task", "status": "pending"}]
await _worker_iteration()
assert worker_module.IN_MEMORY_JOBS[0]["status"] == "completed"


@pytest.mark.asyncio
async def test_check_db_readiness_no_pool() -> None:
worker_module.db_pool = None
Expand Down