Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@ history, so they summarise each release rather than list every change. The
worker processes and returns their arrival-time maps;
`pyforefire.burn_probability` turns them into a burn-probability map. A
User Guide page shows a wind ensemble end to end. ([#172])
- `pyforefire.dice`, `pyforefire.brier_score` and `pyforefire.reliability`
score an ensemble against an observed burnt area: the overlap of each
member, the error of the probability map, and how often each probability
bin actually burnt. ([#172])
- `ForeFireRunner.fork()` returns a new runner holding a copy of the
simulation, so ensemble members can branch from one spin-up instead of
rerunning it. The worker forks itself, which takes a few milliseconds where
Expand Down
2 changes: 1 addition & 1 deletion bindings/python/src/pyforefire/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,6 @@
from .helpers import *
from ._pyforefire import * # Import the C++ extension
from ._runner import ForeFireRunner, WorkerDied
from ._ensemble import run_ensemble, burn_probability, arrival_time
from ._ensemble import run_ensemble, burn_probability, arrival_time, dice, brier_score, reliability

__all__ = ['helpers', '_pyforefire']
59 changes: 58 additions & 1 deletion bindings/python/src/pyforefire/_ensemble.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ def perturb(ff, wind):

arrivals = run_ensemble(setup, [(3, 0), (2, 2), (0, 3)], until=3600, perturb=perturb)
probability = burn_probability(arrivals, t=1800)

`dice`, `brier_score` and `reliability` score the maps against an observed
burnt area.
"""

import os
Expand All @@ -25,7 +28,8 @@ def perturb(ff, wind):

from ._runner import ForeFireRunner

__all__ = ["run_ensemble", "burn_probability", "arrival_time"]
__all__ = ["run_ensemble", "burn_probability", "arrival_time",
"dice", "brier_score", "reliability"]


def arrival_time(ff, path, bbox=None):
Expand Down Expand Up @@ -111,3 +115,56 @@ def burn_probability(arrivals, t=None):
arrivals = np.asarray(arrivals)
burnt = np.isfinite(arrivals) if t is None else arrivals <= t
return burnt.mean(axis=0)


def dice(burnt, observed):
"""Return the Dice overlap 2|A & B| / (|A| + |B|) of two burnt areas.

`burnt` and `observed` are boolean maps indexed [y, x], or `burnt` is
indexed [member, y, x] for one score per member, such as
`np.isfinite(arrivals)` or `arrivals <= t`. 1 is a perfect match, 0 no
overlap. Two empty areas match.
"""
burnt = np.asarray(burnt, dtype=bool)
observed = np.asarray(observed, dtype=bool)
both = (burnt & observed).sum(axis=(-2, -1))
total = burnt.sum(axis=(-2, -1)) + observed.sum()
return np.where(total > 0, 2 * both / np.maximum(total, 1), 1.0)[()]


def brier_score(probability, observed):
"""Return the mean squared difference between a probability map and what burnt.

`probability` is indexed [y, x], as from `burn_probability`, or
[member, y, x]; `observed` is a boolean map indexed [y, x]. 0 is a
perfect forecast. A single member's burnt area scores the fraction of
cells it gets wrong, so an ensemble can be compared with its members.
"""
probability = np.asarray(probability, dtype=float)
observed = np.asarray(observed, dtype=bool)
return ((probability - observed) ** 2).mean(axis=(-2, -1))[()]


def reliability(probability, observed, bins=(0, 0.1, 0.3, 0.5, 0.7, 0.9, 1)):
"""Return how often the cells in each probability bin actually burnt.

`probability` and `observed` are maps of the same shape; to pool several
observation times, stack or concatenate them. `bins` are the edges of the
probability bins; the last bin includes its upper edge.

Returns `(frequency, count)`, one value per bin: the fraction of its
cells observed burnt (`nan` for an empty bin) and its number of cells.
For a reliable ensemble, the frequency lies within its bin.
"""
probability = np.asarray(probability, dtype=float).ravel()
observed = np.asarray(observed, dtype=bool).ravel()
if probability.shape != observed.shape:
raise ValueError("probability and observed must have the same shape")
edges = np.asarray(bins, dtype=float)
index = np.clip(np.searchsorted(edges, probability, side="right") - 1, 0, len(edges) - 2)
inside = (probability >= edges[0]) & (probability <= edges[-1])
count = np.bincount(index[inside], minlength=len(edges) - 1)
hits = np.bincount(index[inside], weights=observed[inside], minlength=len(edges) - 1)
with np.errstate(invalid="ignore", divide="ignore"):
frequency = np.where(count > 0, hits / count, np.nan)
return frequency, count
27 changes: 27 additions & 0 deletions docs/source/user_guide/ensembles.rst
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,33 @@ To plot it:
origin="lower", extent=(0, L, 0, L), colors="cyan")
fig.colorbar(im, label="burn probability")

Scoring against an observed fire
--------------------------------

Given the observed burnt area at time ``t`` as a boolean map ``observed`` on
the same grid (``[y, x]``, row 0 at the south edge), three helpers score the
ensemble:

.. code-block:: python

p = pyforefire.burn_probability(arrivals, t)
pyforefire.dice(arrivals <= t, observed) # overlap of each member, 0 to 1
pyforefire.brier_score(p, observed) # error of the probability map, 0 is perfect
frequency, count = pyforefire.reliability(p, observed)

* ``dice`` is 2|A ∩ B| / (|A| + |B|), one value per member.
* ``brier_score`` is the mean of ``(p - observed)²``. Applied to
``arrivals <= t`` it scores each member alone, so the ensemble can be
compared with its members.
* ``reliability`` gives, for each probability bin (by default 0–0.1, 0.1–0.3,
…, 0.9–1), the fraction of its cells that actually burnt. For a reliable
ensemble, that fraction lies within the bin: about half of the cells given
0.3–0.7 burnt. Stack the maps of several observation times to pool them.

An observed perimeter can be turned into such a map with
``rasterio.features.rasterize``; flip it with ``np.flipud`` if it comes north
row first.

What a member may change
------------------------

Expand Down
48 changes: 46 additions & 2 deletions tests/python/test_ensemble.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Tests for `pyforefire.run_ensemble` and `pyforefire.burn_probability`.
"""Tests for `pyforefire.run_ensemble`, `pyforefire.burn_probability` and the scores.

python tests/python/test_ensemble.py

Expand Down Expand Up @@ -48,6 +48,11 @@ def test_identical_members(reference):
p = pyforefire.burn_probability(arrivals)
if not np.array_equal(p, np.isfinite(reference).astype(float)):
failures.append(f"fork={fork}: probability is not the deterministic burnt area")
observed = np.isfinite(reference)
if not np.all(pyforefire.dice(np.isfinite(arrivals), observed) == 1):
failures.append(f"fork={fork}: a member does not score Dice 1 against the deterministic run")
if pyforefire.brier_score(p, observed) != 0:
failures.append(f"fork={fork}: Brier score not 0 against the deterministic run")
return failures


Expand Down Expand Up @@ -120,6 +125,44 @@ def fail(ff, member):
return ["a failing member did not raise"]


def test_scores():
"""Dice, Brier and reliability on small hand-made maps."""
import pyforefire

failures = []
observed = np.zeros((4, 4), dtype=bool)
observed[:2] = True # 8 cells
burnt = np.zeros((2, 4, 4), dtype=bool)
burnt[0, :2] = True # the observed area
burnt[1, 1:3] = True # half of it, plus 4 cells
d = pyforefire.dice(burnt, observed)
if not np.allclose(d, [1.0, 0.5]):
failures.append(f"Dice per member {d}, expected [1, 0.5]")
if pyforefire.dice(burnt[0], observed) != 1.0:
failures.append("Dice of one map is not a single 1")
empty = np.zeros((4, 4), dtype=bool)
if pyforefire.dice(empty, empty) != 1.0:
failures.append("two empty areas do not match")

p = burnt.mean(axis=0) # 1 on row 1, 0.5 on rows 0 and 2
b = pyforefire.brier_score(p, observed)
if not np.isclose(b, (4 * 0.25 + 4 * 0.25) / 16):
failures.append(f"Brier score {b}, expected 0.125")
bm = pyforefire.brier_score(burnt, observed)
if not np.allclose(bm, [0, 8 / 16]):
failures.append(f"Brier score per member {bm}, expected [0, 0.5]")

frequency, count = pyforefire.reliability(p, observed, bins=(0, 0.25, 0.75, 1))
if list(count) != [4, 8, 4]:
failures.append(f"reliability counts {list(count)}, expected [4, 8, 4]")
elif not np.allclose(frequency, [0, 0.5, 1]):
failures.append(f"reliability frequencies {frequency}, expected [0, 0.5, 1]")
frequency, count = pyforefire.reliability(np.ones(3), np.ones(3, dtype=bool), bins=(0, 0.5, 1))
if list(count) != [0, 3] or not np.isnan(frequency[0]):
failures.append(f"probability 1 not in the last bin, or empty bin not nan: {frequency}, {count}")
return failures


def main():
reference = deterministic()
print(f"deterministic run: {int(np.isfinite(reference).sum())} burnt cells of {reference.size}")
Expand All @@ -130,13 +173,14 @@ def main():
failures += test_construction_parameters()
failures += test_probability_properties()
failures += test_member_error()
failures += test_scores()

if failures:
print(f"\nFAILED with {len(failures)} problem(s):")
for f in failures:
print(f" - {f}")
return 1
print("OK: ensembles reproduce the deterministic run, perturbations apply, probabilities are consistent")
print("OK: ensembles reproduce the deterministic run, perturbations apply, probabilities and scores are consistent")
return 0


Expand Down
Loading