From e81225b4687bffc049bad4803e73244160306f2b Mon Sep 17 00:00:00 2001 From: HugoFara Date: Fri, 30 Oct 2026 22:06:09 +0100 Subject: [PATCH] feat(python): score an ensemble against an observed burnt area (#172) dice, brier_score and reliability take plain arrays: the members' burnt areas or the burn-probability map, and a boolean map of what burnt. Dice and Brier give one value per member on [member, y, x] input, so the ensemble can be compared with its members. --- CHANGELOG.md | 4 ++ bindings/python/src/pyforefire/__init__.py | 2 +- bindings/python/src/pyforefire/_ensemble.py | 59 ++++++++++++++++++++- docs/source/user_guide/ensembles.rst | 27 ++++++++++ tests/python/test_ensemble.py | 48 ++++++++++++++++- 5 files changed, 136 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28b740ec..73133a4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ history, so they summarise each release rather than list every change. The worker processes and returns their arrival-time maps; `pyforefire.burn_probability` turns them into a burn-probability map. A User Guide page shows a wind ensemble end to end. ([#172]) +- `pyforefire.dice`, `pyforefire.brier_score` and `pyforefire.reliability` + score an ensemble against an observed burnt area: the overlap of each + member, the error of the probability map, and how often each probability + bin actually burnt. ([#172]) - `ForeFireRunner.fork()` returns a new runner holding a copy of the simulation, so ensemble members can branch from one spin-up instead of rerunning it. The worker forks itself, which takes a few milliseconds where diff --git a/bindings/python/src/pyforefire/__init__.py b/bindings/python/src/pyforefire/__init__.py index 5931fa2c..80fb93f4 100644 --- a/bindings/python/src/pyforefire/__init__.py +++ b/bindings/python/src/pyforefire/__init__.py @@ -3,6 +3,6 @@ from .helpers import * from ._pyforefire import * # Import the C++ extension from ._runner import ForeFireRunner, WorkerDied -from ._ensemble import run_ensemble, burn_probability, arrival_time +from ._ensemble import run_ensemble, burn_probability, arrival_time, dice, brier_score, reliability __all__ = ['helpers', '_pyforefire'] \ No newline at end of file diff --git a/bindings/python/src/pyforefire/_ensemble.py b/bindings/python/src/pyforefire/_ensemble.py index ad096ca2..896ecf51 100644 --- a/bindings/python/src/pyforefire/_ensemble.py +++ b/bindings/python/src/pyforefire/_ensemble.py @@ -14,6 +14,9 @@ def perturb(ff, wind): arrivals = run_ensemble(setup, [(3, 0), (2, 2), (0, 3)], until=3600, perturb=perturb) probability = burn_probability(arrivals, t=1800) + +`dice`, `brier_score` and `reliability` score the maps against an observed +burnt area. """ import os @@ -25,7 +28,8 @@ def perturb(ff, wind): from ._runner import ForeFireRunner -__all__ = ["run_ensemble", "burn_probability", "arrival_time"] +__all__ = ["run_ensemble", "burn_probability", "arrival_time", + "dice", "brier_score", "reliability"] def arrival_time(ff, path, bbox=None): @@ -111,3 +115,56 @@ def burn_probability(arrivals, t=None): arrivals = np.asarray(arrivals) burnt = np.isfinite(arrivals) if t is None else arrivals <= t return burnt.mean(axis=0) + + +def dice(burnt, observed): + """Return the Dice overlap 2|A & B| / (|A| + |B|) of two burnt areas. + + `burnt` and `observed` are boolean maps indexed [y, x], or `burnt` is + indexed [member, y, x] for one score per member, such as + `np.isfinite(arrivals)` or `arrivals <= t`. 1 is a perfect match, 0 no + overlap. Two empty areas match. + """ + burnt = np.asarray(burnt, dtype=bool) + observed = np.asarray(observed, dtype=bool) + both = (burnt & observed).sum(axis=(-2, -1)) + total = burnt.sum(axis=(-2, -1)) + observed.sum() + return np.where(total > 0, 2 * both / np.maximum(total, 1), 1.0)[()] + + +def brier_score(probability, observed): + """Return the mean squared difference between a probability map and what burnt. + + `probability` is indexed [y, x], as from `burn_probability`, or + [member, y, x]; `observed` is a boolean map indexed [y, x]. 0 is a + perfect forecast. A single member's burnt area scores the fraction of + cells it gets wrong, so an ensemble can be compared with its members. + """ + probability = np.asarray(probability, dtype=float) + observed = np.asarray(observed, dtype=bool) + return ((probability - observed) ** 2).mean(axis=(-2, -1))[()] + + +def reliability(probability, observed, bins=(0, 0.1, 0.3, 0.5, 0.7, 0.9, 1)): + """Return how often the cells in each probability bin actually burnt. + + `probability` and `observed` are maps of the same shape; to pool several + observation times, stack or concatenate them. `bins` are the edges of the + probability bins; the last bin includes its upper edge. + + Returns `(frequency, count)`, one value per bin: the fraction of its + cells observed burnt (`nan` for an empty bin) and its number of cells. + For a reliable ensemble, the frequency lies within its bin. + """ + probability = np.asarray(probability, dtype=float).ravel() + observed = np.asarray(observed, dtype=bool).ravel() + if probability.shape != observed.shape: + raise ValueError("probability and observed must have the same shape") + edges = np.asarray(bins, dtype=float) + index = np.clip(np.searchsorted(edges, probability, side="right") - 1, 0, len(edges) - 2) + inside = (probability >= edges[0]) & (probability <= edges[-1]) + count = np.bincount(index[inside], minlength=len(edges) - 1) + hits = np.bincount(index[inside], weights=observed[inside], minlength=len(edges) - 1) + with np.errstate(invalid="ignore", divide="ignore"): + frequency = np.where(count > 0, hits / count, np.nan) + return frequency, count diff --git a/docs/source/user_guide/ensembles.rst b/docs/source/user_guide/ensembles.rst index 9baf478e..8e481780 100644 --- a/docs/source/user_guide/ensembles.rst +++ b/docs/source/user_guide/ensembles.rst @@ -76,6 +76,33 @@ To plot it: origin="lower", extent=(0, L, 0, L), colors="cyan") fig.colorbar(im, label="burn probability") +Scoring against an observed fire +-------------------------------- + +Given the observed burnt area at time ``t`` as a boolean map ``observed`` on +the same grid (``[y, x]``, row 0 at the south edge), three helpers score the +ensemble: + +.. code-block:: python + + p = pyforefire.burn_probability(arrivals, t) + pyforefire.dice(arrivals <= t, observed) # overlap of each member, 0 to 1 + pyforefire.brier_score(p, observed) # error of the probability map, 0 is perfect + frequency, count = pyforefire.reliability(p, observed) + +* ``dice`` is 2|A ∩ B| / (|A| + |B|), one value per member. +* ``brier_score`` is the mean of ``(p - observed)²``. Applied to + ``arrivals <= t`` it scores each member alone, so the ensemble can be + compared with its members. +* ``reliability`` gives, for each probability bin (by default 0–0.1, 0.1–0.3, + …, 0.9–1), the fraction of its cells that actually burnt. For a reliable + ensemble, that fraction lies within the bin: about half of the cells given + 0.3–0.7 burnt. Stack the maps of several observation times to pool them. + +An observed perimeter can be turned into such a map with +``rasterio.features.rasterize``; flip it with ``np.flipud`` if it comes north +row first. + What a member may change ------------------------ diff --git a/tests/python/test_ensemble.py b/tests/python/test_ensemble.py index d1e8deee..d3a30c28 100644 --- a/tests/python/test_ensemble.py +++ b/tests/python/test_ensemble.py @@ -1,4 +1,4 @@ -"""Tests for `pyforefire.run_ensemble` and `pyforefire.burn_probability`. +"""Tests for `pyforefire.run_ensemble`, `pyforefire.burn_probability` and the scores. python tests/python/test_ensemble.py @@ -48,6 +48,11 @@ def test_identical_members(reference): p = pyforefire.burn_probability(arrivals) if not np.array_equal(p, np.isfinite(reference).astype(float)): failures.append(f"fork={fork}: probability is not the deterministic burnt area") + observed = np.isfinite(reference) + if not np.all(pyforefire.dice(np.isfinite(arrivals), observed) == 1): + failures.append(f"fork={fork}: a member does not score Dice 1 against the deterministic run") + if pyforefire.brier_score(p, observed) != 0: + failures.append(f"fork={fork}: Brier score not 0 against the deterministic run") return failures @@ -120,6 +125,44 @@ def fail(ff, member): return ["a failing member did not raise"] +def test_scores(): + """Dice, Brier and reliability on small hand-made maps.""" + import pyforefire + + failures = [] + observed = np.zeros((4, 4), dtype=bool) + observed[:2] = True # 8 cells + burnt = np.zeros((2, 4, 4), dtype=bool) + burnt[0, :2] = True # the observed area + burnt[1, 1:3] = True # half of it, plus 4 cells + d = pyforefire.dice(burnt, observed) + if not np.allclose(d, [1.0, 0.5]): + failures.append(f"Dice per member {d}, expected [1, 0.5]") + if pyforefire.dice(burnt[0], observed) != 1.0: + failures.append("Dice of one map is not a single 1") + empty = np.zeros((4, 4), dtype=bool) + if pyforefire.dice(empty, empty) != 1.0: + failures.append("two empty areas do not match") + + p = burnt.mean(axis=0) # 1 on row 1, 0.5 on rows 0 and 2 + b = pyforefire.brier_score(p, observed) + if not np.isclose(b, (4 * 0.25 + 4 * 0.25) / 16): + failures.append(f"Brier score {b}, expected 0.125") + bm = pyforefire.brier_score(burnt, observed) + if not np.allclose(bm, [0, 8 / 16]): + failures.append(f"Brier score per member {bm}, expected [0, 0.5]") + + frequency, count = pyforefire.reliability(p, observed, bins=(0, 0.25, 0.75, 1)) + if list(count) != [4, 8, 4]: + failures.append(f"reliability counts {list(count)}, expected [4, 8, 4]") + elif not np.allclose(frequency, [0, 0.5, 1]): + failures.append(f"reliability frequencies {frequency}, expected [0, 0.5, 1]") + frequency, count = pyforefire.reliability(np.ones(3), np.ones(3, dtype=bool), bins=(0, 0.5, 1)) + if list(count) != [0, 3] or not np.isnan(frequency[0]): + failures.append(f"probability 1 not in the last bin, or empty bin not nan: {frequency}, {count}") + return failures + + def main(): reference = deterministic() print(f"deterministic run: {int(np.isfinite(reference).sum())} burnt cells of {reference.size}") @@ -130,13 +173,14 @@ def main(): failures += test_construction_parameters() failures += test_probability_properties() failures += test_member_error() + failures += test_scores() if failures: print(f"\nFAILED with {len(failures)} problem(s):") for f in failures: print(f" - {f}") return 1 - print("OK: ensembles reproduce the deterministic run, perturbations apply, probabilities are consistent") + print("OK: ensembles reproduce the deterministic run, perturbations apply, probabilities and scores are consistent") return 0