diff --git a/.github/workflows/invariants.yml b/.github/workflows/invariants.yml index e25d4dec..800a4b38 100644 --- a/.github/workflows/invariants.yml +++ b/.github/workflows/invariants.yml @@ -69,3 +69,9 @@ jobs: - name: Run the ensemble test run: ./.venv/bin/python tests/python/test_ensemble.py + + - name: Run the FFTile and FFState test + run: ./.venv/bin/python tests/python/test_fftile_ffstate.py + + - name: Run the listen mode test + run: ./.venv/bin/python tests/python/test_listen.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 095c419d..bb319634 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -111,6 +111,9 @@ history, so they summarise each release rather than list every change. The ### Fixed +- `setParameter[key=]` with an empty value no longer crashes. +- `listenHTTP` reads a POST body sent after its headers, as Python's `urllib` does. +- A `listenHTTP` server started from `pyforefire` no longer crashes on its first command. - The Farsite model now matches the FARSITE reference (mbedward/farsite) at the flanks and back of the fire, not only at the head. On slopes it is an approximation. This changes Farsite results. ([#270]) diff --git a/bindings/java/JLibForeFire.cpp b/bindings/java/JLibForeFire.cpp index 15abee1a..25eb68ac 100644 --- a/bindings/java/JLibForeFire.cpp +++ b/bindings/java/JLibForeFire.cpp @@ -47,7 +47,7 @@ JNIEXPORT jstring JNICALL Java_fprop_coupling_CCoupling_executeCommand (JNIEnv } ostringstream stringOut; - executor->setOstringstream(&stringOut); + Command::OutputCapture capture(&stringOut); string smsg(msg); diff --git a/bindings/python/src/pyforefire/_pyforefire.cpp b/bindings/python/src/pyforefire/_pyforefire.cpp index b1d6d8ef..9d7bda96 100644 --- a/bindings/python/src/pyforefire/_pyforefire.cpp +++ b/bindings/python/src/pyforefire/_pyforefire.cpp @@ -135,7 +135,7 @@ py::object PLibForeFire::getDataMatrixPy(char *name) { string PLibForeFire::execute(char *command) { ostringstream stringOut; - pyxecutor->setOstringstream(&stringOut); + Command::OutputCapture capture(&stringOut); string smsg(command); pyxecutor->ExecuteCommand(smsg); return stringOut.str(); diff --git a/bindings/wasm/forefire_wasm.cpp b/bindings/wasm/forefire_wasm.cpp index bbd47c5f..9ad95f10 100644 --- a/bindings/wasm/forefire_wasm.cpp +++ b/bindings/wasm/forefire_wasm.cpp @@ -77,7 +77,7 @@ class ForeFireWasm { /*! \brief Run one ForeFire command, returning whatever it printed. */ std::string execute(const std::string &command) { std::ostringstream captured; - executor.setOstringstream(&captured); + Command::OutputCapture capture(&captured); std::string cmd(command); executor.ExecuteCommand(cmd); return captured.str(); diff --git a/src/Command.cpp b/src/Command.cpp index cf73b5ff..e8e38152 100644 --- a/src/Command.cpp +++ b/src/Command.cpp @@ -2051,11 +2051,14 @@ namespace libforefire vector tmpArgs; string delimiter = "="; tokenize(arg, tmpArgs, delimiter); - if (tmpArgs.size() > 2) + if (tmpArgs.empty() || tmpArgs.size() > 2 + || arg.find(delimiter) == string::npos || arg.compare(0, delimiter.size(), delimiter) == 0) { cout << "problem in the number of arguments when setting " << arg << endl; return error; } + // "key=" sets an empty value; tokenize drops the empty token. + if (tmpArgs.size() == 1) tmpArgs.push_back(""); // Check if the parameter value begins with '!' if (!tmpArgs[1].empty() && tmpArgs[1][0] == '!') @@ -3117,7 +3120,8 @@ namespace libforefire void Command::setOstringstream(ostringstream *oss) { - currentSession.outStream = oss; + // nullptr goes back to cout, for callers whose stream is going away. + currentSession.outStream = oss ? static_cast(oss) : &cout; } void Command::tokenize(const string &str, vector &tokens, diff --git a/src/Command.h b/src/Command.h index 701935be..194c248c 100644 --- a/src/Command.h +++ b/src/Command.h @@ -319,6 +319,16 @@ class Command { static bool domainExists(const string& command); /*! \brief command to redirect the ostringstream */ static void setOstringstream(ostringstream*); + /*! \brief sends command output to a stream until destroyed, then back to + * cout, so later output (e.g. from a listenHTTP thread) never reaches a + * stream that is gone */ + class OutputCapture { + public: + explicit OutputCapture(ostringstream* out) { setOstringstream(out); } + ~OutputCapture() { setOstringstream(nullptr); } + OutputCapture(const OutputCapture&) = delete; + OutputCapture& operator=(const OutputCapture&) = delete; + }; /*! \brief execute the desired command */ static void ExecuteCommand(string&); diff --git a/src/HttpCommandServer.hpp b/src/HttpCommandServer.hpp index eeea1456..9010f398 100644 --- a/src/HttpCommandServer.hpp +++ b/src/HttpCommandServer.hpp @@ -19,6 +19,7 @@ #include #include #include +#include #include // POSIX headers for sockets @@ -151,20 +152,50 @@ namespace http_command { } continue; } + // Clients are served one at a time: do not let a stalled one + // hold the server. + timeval timeout{5, 0}; + setsockopt(client_fd, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); handleClient(client_fd); close(client_fd); } } + // Read the whole request: headers, then Content-Length bytes of body. + // Clients may send them in separate segments, so one read() is not + // enough. Empty on a closed connection or an oversized request. + static std::string readRequest(int client_fd) { + const size_t max_size = 16 << 20; + std::string request; + char buffer[4096]; + size_t header_end = std::string::npos, expected = 0; + while (true) { + if (header_end == std::string::npos) { + header_end = request.find("\r\n\r\n"); + if (header_end != std::string::npos) { + std::string length = headerValue(request, "content-length"); + size_t body = 0; + try { body = length.empty() ? 0 : std::stoull(length); } + catch (...) { return ""; } + if (body > max_size) return ""; + expected = header_end + 4 + body; + } + } + if (header_end != std::string::npos && request.size() >= expected) + return request; + if (request.size() > max_size) return ""; + ssize_t n = read(client_fd, buffer, sizeof(buffer)); + if (n <= 0) + return header_end == std::string::npos ? request : ""; + request.append(buffer, static_cast(n)); + } + } + // Handle a client connection by parsing the request. void handleClient(int client_fd) { - const int buf_size = 4096; - char buffer[buf_size]; - int bytes_read = read(client_fd, buffer, buf_size - 1); - if (bytes_read <= 0) + std::string request = readRequest(client_fd); + if (request.empty()) return; - buffer[bytes_read] = '\0'; - std::string request(buffer); // Parse the request line (e.g., "GET /uri HTTP/1.1" or "POST /uri HTTP/1.1"). std::istringstream requestStream(request); diff --git a/tests/python/README.md b/tests/python/README.md index 2b83c629..6a382227 100644 --- a/tests/python/README.md +++ b/tests/python/README.md @@ -94,6 +94,28 @@ decrease with time. Run in CI by `invariants.yml`. Manually: python tests/python/test_ensemble.py ``` +### `test_fftile_ffstate.py` + +Writes one small synthetic `.fftile` with `tools/tiles/fftile.py`, loads it +from Python and checks the fuel and altitude layers. A checkpoint saved in one +worker and restored in a fresh one must then continue exactly like the +original, for both arrival-map layouts. A corrupt checkpoint must be refused. +Run in CI by `invariants.yml`. Manually: + +```bash +python tests/python/test_fftile_ffstate.py +``` + +### `test_listen.py` + +Starts `listenHTTP` in a worker and drives a tile case over HTTP, by GET and +POST, including the commands the web page sends. The checkpoint it saves must +restore in a fresh worker. Run in CI by `invariants.yml`. Manually: + +```bash +python tests/python/test_listen.py +``` + ### `test_threading.py` Runs eight simulations in threads and requires each to reproduce the node diff --git a/tests/python/test_fftile_ffstate.py b/tests/python/test_fftile_ffstate.py new file mode 100644 index 00000000..e2966a6f --- /dev/null +++ b/tests/python/test_fftile_ffstate.py @@ -0,0 +1,163 @@ +"""FFTile landscapes and FFState checkpoints driven from `pyforefire`. + +Writes one small synthetic tile, so it needs no download. Each simulation runs +in its own `ForeFireRunner` worker: the kernel keeps one domain per process, +and a checkpoint restore needs an empty session. + + python tests/python/test_fftile_ffstate.py + +Exit status is 0 only if every check passes. +""" + +import sys +import tempfile +import traceback +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tools" / "tiles")) +sys.path.insert(0, str(ROOT / "tests" / "ffstate")) +from fftile import write_tile # noqa: E402 +from smoke import chunks # noqa: E402 + +from pyforefire import ForeFireRunner # noqa: E402 + +FUEL = 2 +ALTITUDE = 120.0 +# Chunks that are the fire state; output settings and bookkeeping are not. +STATE_CHUNKS = ("domain", "objects.ff", "events", "arrival") + + +def write_landscape(directory): + flat = np.zeros((16, 16), np.float32) + fuel = np.full((64, 64), FUEL, np.uint8) + write_tile(directory / "42_8.5.fftile", 42, 8.5, fuel, flat + ALTITUDE, + flat, flat, flat + 1, flat, flat, flat + 1, timestamp=0) + + +def session(case, subdivisions): + """A worker with the case parameters set and no domain yet.""" + ff = ForeFireRunner() + # An empty value: setParameter must not read past its argument list. + ff.execute(f"setParameters[caseDirectory={case};ForeFireDataDirectory=]") + ff.execute("setParameters[propagationModel=Rothermel;" + "fuelsTable=STDlandIndexedFuelsTable;perimeterResolution=24;" + "spatialIncrement=6;minSpeed=0;outputsUpdate=0;bmapLayer=1;" + f"arrivalMapSubdivisions={subdivisions}]") + return ff + + +def ignite(ff, tiles): + ff.execute(f"loadData[provider={tiles};center=(8.8,42.15);size=(.08,.08);" + "date=2025-06-01T12:00:00Z]") + ff.execute("trigger[wind;vel=(6,2,0)]") + ff.execute("startFire[lonlat=(8.8,42.15,0);t=0]") + + +def same_state(a, b): + left, right = chunks(a), chunks(b) + return [name for name in STATE_CHUNKS if left[name] != right[name]] + + +def check_tile_layers(case, tiles): + ff = session(case, 0) + try: + ignite(ff, tiles) + fuel = ff.getDoubleArray("fuel") + altitude = ff.getDoubleArray("altitude") + assert fuel.size and np.all(fuel == FUEL), "fuel layer does not hold the tile's fuel" + assert altitude.size and np.allclose(altitude, ALTITUDE), "altitude layer does not hold the tile's" + ff.execute("step[dt=300]") + arrival = ff.getDoubleArray("BMap") + assert np.isfinite(arrival).sum() > 0, "no arrival time after 300 s of spread" + assert ff.execute("print[]").count("FireNode") > 0 + finally: + ff.close() + + +def check_restore(case, tiles, subdivisions): + """A checkpoint restored in a fresh worker continues like the original.""" + case = case / f"subdivisions-{subdivisions}" + case.mkdir() + seed, continuous, resumed = (case / name for name in + ("seed.ffstate", "continuous.ffstate", "resumed.ffstate")) + original = session(case, subdivisions) + restored = session(case, subdivisions) + try: + ignite(original, tiles) + original.execute("step[dt=120.25]") + original.execute(f"saveState[file={seed}]") + original.execute("step[dt=300]") + original.execute(f"saveState[file={continuous}]") + + out = restored.execute(f"loadState[file={seed}]") + assert "Loaded FFState" in out, out + restored.execute("step[dt=300]") + restored.execute(f"saveState[file={resumed}]") + + differ = same_state(continuous, resumed) + assert not differ, f"restored run differs from the original in {differ}" + assert np.array_equal(original.getDoubleArray("BMap"), restored.getDoubleArray("BMap"), + equal_nan=True), "arrival times read from Python differ" + assert original.execute("print[]") == restored.execute("print[]") + + # A loaded session is never replaced, and stays usable. + out = restored.execute(f"loadState[file={seed}]") + assert "domain exists" in out, out + restored.execute("step[dt=60]") + finally: + original.close() + restored.close() + + +def check_corrupt_state(case, tiles): + ff = session(case, 0) + try: + ignite(ff, tiles) + ff.execute("step[dt=60]") + good = case / "good.ffstate" + ff.execute(f"saveState[file={good}]") + finally: + ff.close() + bad = bytearray(good.read_bytes()) + bad[-1] ^= 1 + (case / "bad.ffstate").write_bytes(bad) + ff = session(case, 0) + try: + out = ff.execute(f"loadState[file={case / 'bad.ffstate'}]") + assert "checksum mismatch" in out, out + # Nothing was published: the good checkpoint still loads. + out = ff.execute(f"loadState[file={good}]") + assert "Loaded FFState" in out, out + finally: + ff.close() + + +def main(): + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + tiles = tmp / "tiles" + tiles.mkdir() + write_landscape(tiles) + checks = [ + ("tile layers read from Python", lambda: check_tile_layers(tmp, tiles)), + ("restore, Cartesian arrival map", lambda: check_restore(tmp, tiles, 0)), + ("restore, geographic arrival map", lambda: check_restore(tmp, tiles, 256)), + ("corrupt checkpoint refused", lambda: check_corrupt_state(tmp, tiles)), + ] + failed = 0 + for name, check in checks: + try: + check() + print(f"PASS {name}") + except Exception: + failed += 1 + print(f"FAIL {name}") + traceback.print_exc() + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/python/test_listen.py b/tests/python/test_listen.py new file mode 100644 index 00000000..9fb7be34 --- /dev/null +++ b/tests/python/test_listen.py @@ -0,0 +1,161 @@ +"""Listen mode (`listenHTTP`) over an FFTile landscape, driven from Python. + +A `ForeFireRunner` worker starts the HTTP command server; the test then sends +the commands a client would, by GET and by POST, while the worker is idle. +Checks that a run driven over HTTP saves a checkpoint that a fresh worker +restores and continues exactly like the served session. + + python tests/python/test_listen.py + +Exit status is 0 only if every check passes. +""" + +import socket +import sys +import tempfile +import time +import traceback +import urllib.parse +import urllib.request +from pathlib import Path + +import numpy as np + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "tests" / "python")) +from test_fftile_ffstate import ignite, same_state, session, write_landscape # noqa: E402 + + +def free_port(): + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +class Client: + def __init__(self, port): + self.base = f"http://127.0.0.1:{port}" + + def get(self, command): + quoted = urllib.parse.quote(command, safe="[]=;,()/:.") + with urllib.request.urlopen(f"{self.base}/ff:{quoted}", timeout=60) as r: + return r.read().decode() + + def post(self, command): + request = urllib.request.Request(f"{self.base}/", data=("ff:" + command).encode()) + with urllib.request.urlopen(request, timeout=60) as r: + return r.read().decode() + + def wait(self): + for _ in range(100): + try: + return self.get("getParameter[ISOdate]") + except OSError: + time.sleep(0.05) + raise RuntimeError("the HTTP server never answered") + + +def split_post(port, command): + """POST whose body arrives after the headers, in its own segment.""" + body = ("ff:" + command).encode() + with socket.create_connection(("127.0.0.1", port)) as c: + c.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + c.sendall(f"POST / HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\n" + f"Content-Length: {len(body)}\r\n\r\n".encode()) + time.sleep(0.05) + c.sendall(body) + return c.makefile("rb").read().decode() + + +def check_served_run(case, tiles): + port = free_port() + served = session(case, 0) + restored = session(case, 0) + try: + served.execute(f"listenHTTP[127.0.0.1:{port}]") + http = Client(port) + http.wait() + + # The landscape and the ignition, as a client would send them. + out = http.get(f"loadData[provider={tiles};center=(8.8,42.15);size=(.08,.08);" + "date=2025-06-01T12:00:00Z]") + assert "1 available" in out, out + http.post("trigger[wind;vel=(6,2,0)]") + http.post("startFire[lonlat=(8.8,42.15,0);t=0]") + assert "FireNode" in http.get("print[]"), "POST startFire[] did not start a fire" + + http.post("step[dt=120.25]") + seed = case / "served.ffstate" + assert "Saved FFState" in http.post(f"saveState[file={seed}]") + assert http.get("getParameter[ISOdate]").startswith("2025-06-01T12:02:00") + + # What the web page reads back on a tile domain. + assert "fuel" in http.get("getParameter[layerNames]") + wind = http.post(f"plot[parameter=wind;filename={case / 'wind.json'};size=8,8;" + "projectionOut=json]") + assert '"SWlon":8.76' in wind, wind + http.post("setParameter[dumpMode=geojson]") + geojson = http.post("print[]") + http.post("setParameter[dumpMode=ff]") + assert '"FeatureCollection"' in geojson and "8.79" in geojson, geojson[:300] + + # The served session is the worker's own one. + assert served.execute("print[]") == http.get("print[]") + + http.post("step[dt=300]") + continuous = case / "continuous.ffstate" + http.post(f"saveState[file={continuous}]") + + out = restored.execute(f"loadState[file={seed}]") + assert "Loaded FFState" in out, out + restored.execute("step[dt=300]") + resumed = case / "resumed.ffstate" + restored.execute(f"saveState[file={resumed}]") + differ = same_state(continuous, resumed) + assert not differ, f"restored run differs from the served one in {differ}" + assert np.array_equal(served.getDoubleArray("BMap"), restored.getDoubleArray("BMap"), + equal_nan=True) + finally: + served.close() + restored.close() + + +def check_split_post(case): + """A body sent after the headers is still read.""" + port = free_port() + ff = session(case, 0) + try: + ff.execute(f"listenHTTP[127.0.0.1:{port}]") + Client(port).wait() + reply = split_post(port, "getParameter[propagationModel]") + assert reply.startswith("HTTP/1.0 200") and "Rothermel" in reply, reply + for _ in range(20): + assert "Rothermel" in Client(port).post("getParameter[propagationModel]") + finally: + ff.close() + + +def main(): + with tempfile.TemporaryDirectory() as tmp: + tmp = Path(tmp) + tiles = tmp / "tiles" + tiles.mkdir() + write_landscape(tiles) + checks = [ + ("run served over HTTP, restored in a fresh worker", lambda: check_served_run(tmp, tiles)), + ("POST body in its own segment", lambda: check_split_post(tmp)), + ] + failed = 0 + for name, check in checks: + try: + check() + print(f"PASS {name}") + except Exception: + failed += 1 + print(f"FAIL {name}") + traceback.print_exc() + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main())