Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .github/workflows/invariants.yml
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,6 @@ jobs:

- name: Run the worker process runner test
run: ./.venv/bin/python tests/python/test_runner.py

- name: Run the ensemble test
run: ./.venv/bin/python tests/python/test_ensemble.py
16 changes: 16 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,21 @@ history, so they summarise each release rather than list every change. The

## [Unreleased]

### Added

- `pyforefire.run_ensemble` runs one simulation per member in parallel
worker processes and returns their arrival-time maps;
`pyforefire.burn_probability` turns them into a burn-probability map. A
User Guide page shows a wind ensemble end to end. ([#172])
- `ForeFireRunner.fork()` returns a new runner holding a copy of the
simulation, so ensemble members can branch from one spin-up instead of
rerunning it. The worker forks itself, which takes a few milliseconds where
a new runner takes about 80. ([#172])
- `ForeFireRunner(stdout=..., stderr=...)` redirects the kernel's output, as
in `subprocess.Popen`. ([#172])
- `pyforefire.arrival_time(ff, path, bbox=None)` reads the arrival-time map
as an array, through the `plot` command. ([#172])

### Fixed

- Commands that need a domain print an error when there is none, instead of
Expand Down Expand Up @@ -303,3 +318,4 @@ repository stays easy to return to.
[#203]: https://github.com/forefireAPI/forefire/pull/203
[#205]: https://github.com/forefireAPI/forefire/issues/205
[#171]: https://github.com/forefireAPI/forefire/issues/171
[#172]: https://github.com/forefireAPI/forefire/issues/172
20 changes: 17 additions & 3 deletions TESTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -116,14 +116,28 @@ for the right reason.
Checks `pyforefire.ForeFireRunner`, which runs each simulation in its own
worker process (#175). Eight runners driven from threads must each reproduce
the in-process fire node count; exceptions in the worker must reach the
caller, and a killed worker must raise `WorkerDied`. It needs no free-threaded
interpreter, because the runners are separate processes. CI runs it in the
Model Invariants workflow.
caller, and a killed worker must raise `WorkerDied`. A fork taken mid-run must
finish with the same count, and changes on either side of a fork must not reach
the other (#172). It needs no free-threaded interpreter, because the runners
are separate processes. CI runs it in the Model Invariants workflow.

```bash
./.venv/bin/python tests/python/test_runner.py
```

## Running the Ensemble Test (`test_ensemble.py`)

Checks `pyforefire.run_ensemble` and `pyforefire.burn_probability` (#172).
Unperturbed members, forked or not, must reproduce the deterministic arrival
times exactly; a perturbation applied to forks must show in the probability
map, and a parameter read at construction must take effect with `fork=False`.
Probabilities must stay in [0, 1] and never decrease with time. CI runs it in
the Model Invariants workflow.

```bash
./.venv/bin/python tests/python/test_ensemble.py
```

## Running the ANN Test (`runANN`)

Validated in CI by `main.yml`. It is the only coverage for
Expand Down
13 changes: 7 additions & 6 deletions bindings/python/src/pyforefire/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# src/pyforefire/__init__.py

from .helpers import *
from ._pyforefire import * # Import the C++ extension
from ._runner import ForeFireRunner, WorkerDied

# src/pyforefire/__init__.py

from .helpers import *
from ._pyforefire import * # Import the C++ extension
from ._runner import ForeFireRunner, WorkerDied
from ._ensemble import run_ensemble, burn_probability, arrival_time

__all__ = ['helpers', '_pyforefire']
113 changes: 113 additions & 0 deletions bindings/python/src/pyforefire/_ensemble.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
"""Run an ensemble of simulations and turn it into a burn-probability map.

Each member runs in its own `ForeFireRunner`, so members run in parallel and
cannot disturb each other (see issue #172).

def setup(ff):
ff.execute("FireDomain[sw=(0,0,0);ne=(1000,1000,0);t=0]")
ff.addLayer("propagation", "Iso", "propagationModel")
ff.execute("startFire[loc=(500,500,0)]")
ff.execute("goTo[t=600]") # spin-up shared by every member

def perturb(ff, wind):
ff.execute(f"trigger[wind;vel=({wind[0]},{wind[1]},0)]")

arrivals = run_ensemble(setup, [(3, 0), (2, 2), (0, 3)], until=3600, perturb=perturb)
probability = burn_probability(arrivals, t=1800)
"""

import os
import subprocess
import tempfile
from concurrent.futures import ThreadPoolExecutor

import numpy as np

from ._runner import ForeFireRunner

__all__ = ["run_ensemble", "burn_probability", "arrival_time"]


def arrival_time(ff, path, bbox=None):
"""Return the arrival-time map of `ff` as an array indexed [y, x].

Row 0 is the south edge, and cells the front has not reached are `inf`.
`bbox=(left, bottom, right, top)` in domain coordinates crops the map to
that area at the same resolution; by default it covers the whole domain.
The map is written to `path` by the kernel's `plot` command, as an ESRI
ASCII grid with the north row first, then read back.
"""
area = "" if bbox is None else ";BBoxLBRT=({},{},{},{})".format(*bbox)
ff.execute(f"plot[parameter=arrival_time;filename={path}{area}]")
return np.flipud(np.loadtxt(path, skiprows=6, ndmin=2))


def _set_parameters(ff, member):
for key, value in member.items():
ff[key] = value


def run_ensemble(setup, members, until, perturb=_set_parameters, fork=True,
bbox=None, max_workers=None, stdout=subprocess.DEVNULL, stderr=None):
"""Run one simulation per item of `members` up to time `until`.

`setup(ff)` builds the simulation: domain, layers, ignition and any
spin-up steps. `perturb(ff, member)` applies one member's changes; by
default `member` is a dict of parameters, set with `ff[key] = value`.

With `fork=True`, `setup` runs once and each member is a fork of it, so
`perturb` runs after `setup`. It can change the wind with
`trigger[wind;vel=(u,v,0)]`, the fuels with `trigger[fuel;...]`, or any
parameter the kernel reads while stepping. With `fork=False`, each member
gets a fresh runner and `perturb` runs before `setup`, so it can also set
the parameters read only when the domain and models are built, such as
`windReductionFactor` and `propagationSpeedAdjustmentFactor`.

`bbox=(left, bottom, right, top)` limits the maps to that area (see
`arrival_time`). The result holds one float64 map per member, so crop a
large domain to where the fire can go.

Members run `max_workers` at a time, one per CPU by default. The kernel's
output is discarded unless `stdout` and `stderr` say otherwise (see
`ForeFireRunner`).

Returns the arrival times as an array indexed [member, y, x], `inf` where
a member's front did not arrive by `until`.
"""
members = list(members)
if max_workers is None:
max_workers = os.cpu_count() or 1

base = None

def run(index, member):
ff = base.fork() if fork else ForeFireRunner(stdout=stdout, stderr=stderr)
with ff:
perturb(ff, member)
if not fork:
setup(ff)
ff.execute(f"goTo[t={until}]")
return arrival_time(ff, os.path.join(tmp, f"member{index}.asc"), bbox)

with tempfile.TemporaryDirectory(prefix="forefire-ensemble-") as tmp:
try:
if fork:
base = ForeFireRunner(stdout=stdout, stderr=stderr)
setup(base)
with ThreadPoolExecutor(max_workers=max_workers) as pool:
maps = list(pool.map(run, range(len(members)), members))
finally:
if base is not None:
base.close()
return np.stack(maps) if maps else np.empty((0, 0, 0))


def burn_probability(arrivals, t=None):
"""Return the fraction of members burnt at each point by time `t`.

`arrivals` is the [member, y, x] array from `run_ensemble`. With `t`
None, a point counts as burnt if the front reached it at all.
"""
arrivals = np.asarray(arrivals)
burnt = np.isfinite(arrivals) if t is None else arrivals <= t
return burnt.mean(axis=0)
103 changes: 96 additions & 7 deletions bindings/python/src/pyforefire/_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,29 @@
The worker is a fresh interpreter (`python -m pyforefire._worker <fd>`) rather
than a fork of the caller, because the caller may have threads. Requests and
replies are pickled over a socket pair. The worker's stdout and stderr are the
caller's, so kernel output still shows up where it did before.
caller's unless redirected, so kernel output still shows up where it did
before.

`fork()` does fork, but the worker rather than the caller: the worker has no
threads, and its copy starts with the whole simulation state, which is what an
ensemble branching from a common spin-up needs (see issue #172).

runner = ForeFireRunner()
runner.execute("FireDomain[sw=(0,0,0);ne=(1000,1000,0);t=0]")
runner["propagationModel"] = "Iso"
copy = runner.fork()
runner.close()
"""

import os
import signal
import socket
import subprocess
import sys
import threading
import time
from multiprocessing.connection import Connection
from multiprocessing.reduction import send_handle

__all__ = ["ForeFireRunner", "WorkerDied"]

Expand All @@ -29,6 +39,41 @@ class WorkerDied(RuntimeError):
"""The worker process ended while a call was pending, or before it."""


class _ForkedProcess:
"""The part of `subprocess.Popen` the runner uses, for a forked worker.

A forked worker is not a child of the caller, so its exit status is
unknown: `poll()` and `wait()` always return None, and `wait()` watches
the process with signal 0 instead of waiting for it.
"""

def __init__(self, pid):
self.pid = pid

def _alive(self):
try:
os.kill(self.pid, 0)
except (ProcessLookupError, PermissionError):
return False
return True

def poll(self):
return None

def wait(self, timeout=None):
deadline = None if timeout is None else time.monotonic() + timeout
while self._alive():
if deadline is not None and time.monotonic() > deadline:
raise subprocess.TimeoutExpired(f"pid {self.pid}", timeout)
time.sleep(0.01)

def kill(self):
try:
os.kill(self.pid, signal.SIGKILL)
except ProcessLookupError:
pass


class ForeFireRunner:
"""A `pyforefire.ForeFire` that lives in a separate process.

Expand All @@ -40,32 +85,46 @@ class ForeFireRunner:

Calls on one runner are serialised by a lock. Separate runners do not
share anything and can be driven from separate threads.

`stdout` and `stderr` redirect the kernel's output as in
`subprocess.Popen`, for example `subprocess.DEVNULL` or an open file.
By default it goes to the caller's own streams. A fork keeps the streams
of the runner it was forked from.
"""

def __init__(self):
def __init__(self, stdout=None, stderr=None):
parent, child = socket.socketpair()
try:
self._process = subprocess.Popen(
process = subprocess.Popen(
[sys.executable, "-m", "pyforefire._worker", str(child.fileno())],
pass_fds=(child.fileno(),),
stdout=stdout,
stderr=stderr,
)
finally:
child.close()
self._conn = Connection(parent.detach())
self._attach(Connection(parent.detach()), process)

def _attach(self, conn, process):
self._conn = conn
self._process = process
self._lock = threading.Lock()

def _call(self, name, *args):
def _call(self, name, *args, fd=None):
with self._lock:
if self._conn is None:
raise WorkerDied("the runner is closed")
try:
self._conn.send((name, args))
if fd is not None:
send_handle(self._conn, fd, self._process.pid)
ok, value = self._conn.recv()
except (EOFError, OSError):
code = self._process.wait()
self._conn.close()
self._conn = None
raise WorkerDied(f"the ForeFire worker exited with status {code}") from None
status = "" if code is None else f" with status {code}"
raise WorkerDied(f"the ForeFire worker exited{status}") from None
if not ok:
raise value
return value
Expand All @@ -81,6 +140,37 @@ def __getitem__(self, key):
def __setitem__(self, key, value):
self._call("__setitem__", key, value)

def fork(self):
"""Return a new runner holding a copy of this simulation.

The worker forks itself, so the copy starts with the same domain,
layers, fronts and parameters, then shares nothing with this runner:
each can be stepped or changed without affecting the other, and
closing one leaves the other running. Memory pages are copied only
when one side writes them, so forking after a long spin-up is cheap.

Files the simulation has open are shared with the copy, and a running
HTTP command server is not copied.
"""
ours, theirs = socket.socketpair()
try:
try:
self._call("__fork__", fd=theirs.fileno())
finally:
theirs.close()
conn = Connection(ours.detach())
except BaseException:
ours.close()
raise
try:
pid = conn.recv()
except (EOFError, OSError):
conn.close()
raise WorkerDied("the forked ForeFire worker exited before it started") from None
copy = ForeFireRunner.__new__(ForeFireRunner)
copy._attach(conn, _ForkedProcess(pid))
return copy

def close(self):
"""Stop the worker and wait for it. Calling it twice is harmless."""
with self._lock:
Expand All @@ -107,4 +197,3 @@ def __exit__(self, *exc):
def __del__(self):
if getattr(self, "_process", None) is not None:
self.close()

Loading
Loading